def get_prices(numdays=1000):
now = Timestamp.utcnow()
bundle = load('quantopian-quandl', os.environ, now)
all_assets = bundle.asset_finder.retrieve_all(bundle.asset_finder.sids)
symbols = set(
str(asset.symbol) for asset in bundle.asset_finder.retrieve_all(
bundle.asset_finder.equities_sids))
quandl_data = DataPortal(
asset_finder=bundle.asset_finder,
trading_calendar=get_calendar('NYSE'),
first_trading_day=bundle.equity_daily_bar_reader.first_trading_day,
equity_minute_reader=bundle.equity_minute_bar_reader,
equity_daily_reader=bundle.equity_daily_bar_reader,
adjustment_reader=bundle.adjustment_reader)
end_dt = pd.Timestamp('2018-03-31', tz='utc')
quandl_symbols = []
for ticker in symbols:
try:
quandl_symbols.append(
quandl_data.asset_finder.lookup_symbol(ticker, end_dt))
except:
continue
quandl_pricing = quandl_data.get_history_window(quandl_symbols, end_dt,
numdays, '1d', 'close',
'daily')
quandl_pricing.columns = [c.symbol for c in quandl_pricing.columns]
return quandl_pricing
def set_bundle(name, calendar='NYSE'):
global trading_calendar
global bundle
global bundle_data
global engine
global choose_loader
global data
bundle = name
trading_calendar = get_calendar(calendar)
bundle_data = bundles.load(bundle)
engine = SimplePipelineEngine(
get_loader=choose_loader,
calendar=trading_calendar.all_sessions,
asset_finder=bundle_data.asset_finder,
)
data = DataPortal(
bundle_data.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=bundle_data.equity_daily_bar_reader.first_trading_day,
equity_minute_reader=None,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
def make_SIC_classifier(bundle='quandl', infile='profiles_20170918.csv'):
bundle_data = bundles.load(bundle)
df_p = pd.read_csv('profiles_20170918.csv')
df_cik = pd.read_csv('cik_ticker_09152017.csv', sep='|')
df_cik['SIC'] = df_cik['SIC'].fillna(-1).astype(np.int64).astype(str)
df_cik['SIC_MajorIndustry'] = df_cik['SIC'].str[:2]
df_cik['SIC_SubClassification'] = df_cik['SIC'].str[:3]
df_cik['SIC_Specialization'] = df_cik['SIC'].str[:4]
df_cik_select = df_cik.loc[df_cik.Ticker.isin(df_p['quandl_sym'])]
tickers = bundle_data.asset_finder.lookup_symbols(df_cik_select['Ticker'],
as_of_date=None)
sids = [asset.sid for asset in tickers]
max_sid = np.max(bundle_data.asset_finder.sids)
major = np.full(max_sid + 1, -1, np.dtype('int64'))
subclass = np.full(max_sid + 1, -1, np.dtype('int64'))
specialize = np.full(max_sid + 1, -1, np.dtype('int64'))
major[sids] = df_cik_select['SIC_MajorIndustry'].astype(np.int64)
subclass[sids] = df_cik_select['SIC_SubClassification'].astype(np.int64)
specialize[sids] = df_cik_select['SIC_Specialization'].astype(np.int64)
np.save('sic_major', major)
np.save('sic_subclass', subclass)
np.save('sic_specialize', specialize)
def _data_portal(bundle=DEFAULT_BUNDLE):
"""Create a DataPortal for the given bundle.
Parameters
----------
bundle : str
The name of the bundle to create a pipeline engine for.
Returns
-------
engine : zipline.pipleine.engine.SimplePipelineEngine
The pipeline engine which can run pipelines against the bundle.
calendar : zipline.utils.calendars.TradingCalendar
The trading calendar for the bundle.
"""
bundle_data = bundles.load(bundle)
calendar = bundle_data.equity_daily_bar_reader.trading_calendar
finder = bundle_data.asset_finder
data_portal = DataPortal(
finder,
calendar,
calendar.first_session,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader)
return data_portal, calendar
def make_sector_classifier(bundle='quandl', infile='profiles_20170918.csv'):
"""
For a given bundle, create the .npy Sector and Industry classifier
files.
"""
bundle_data = bundles.load(bundle)
df_p = pd.read_csv(infile)
labels_sector, uniques_sector = pd.factorize(df_p['sector'])
labels_industry, uniques_industry = pd.factorize(df_p['industry'])
tickers = bundle_data.asset_finder.lookup_symbols(df_p['quandl_sym'],
as_of_date=None)
sids = [asset.sid for asset in tickers]
max_sid = np.max(bundle_data.asset_finder.sids)
sectors = np.full(np.max(max_sid) + 1, -1, np.dtype('int64'))
industries = np.full(np.max(max_sid) + 1, -1, np.dtype('int64'))
sectors[sids] = labels_sector
industries[sids] = labels_industry
np.save('sectors', sectors)
np.save('industries', industries)
pd.DataFrame(data=uniques_sector.tolist()).to_csv('sector_names.csv',
header=False)
pd.DataFrame(data=uniques_industry.tolist()).to_csv('industry_names.csv',
header=False)
return True
def test_bundle(self):
environ = {
'CSVDIR': test_resource_path('csvdir_samples', 'csvdir')
}
ingest('csvdir', environ=environ)
bundle = load('csvdir', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, 'bfill')],
sessions[sessions.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder,
)
assert_equal(actual, expected_pricing, array_decimal=2)
adjs_for_cols = bundle.adjustment_reader.load_pricing_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
assert_equal([sorted(adj.keys()) for adj in adjs_for_cols],
expected_adjustments)
def test_bundle(self):
# # 耗时3秒以内
ingest(TEST_BUNDLE_NAME)
bundle = load(TEST_BUNDLE_NAME)
sids = TEST_SIDS
assert_equal(set(bundle.asset_finder.sids), set(sids))
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.start_date, 'bfill')],
sessions[sessions.get_loc(self.end_date, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder, )
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def make_pipeline_engine(symbols=['SPY', 'TLT'], bundle='etfs_bundle', calendar='NYSE'):
register(bundle, symbols)
bundle_data = load(bundle)
# Set up pipeline engine
# Loader for pricing
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def my_dispatcher(column):
return loaders[column]
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
return my_dispatcher(column)
trading_calendar = get_calendar(calendar)
engine = SimplePipelineEngine(
get_loader=choose_loader,
calendar=trading_calendar.all_sessions,
asset_finder=bundle_data.asset_finder,
)
assets = bundle_data.asset_finder.lookup_symbols(symbols, as_of_date=None)
return assets, engine
def _data_portal(bundle=DEFAULT_BUNDLE):
"""Create a DataPortal for the given bundle.
Parameters
----------
bundle : str
The name of the bundle to create a pipeline engine for.
Returns
-------
engine : zipline.pipleine.engine.SimplePipelineEngine
The pipeline engine which can run pipelines against the bundle.
calendar : zipline.utils.calendars.TradingCalendar
The trading calendar for the bundle.
"""
bundle_data = bundles.load(bundle)
calendar = bundle_data.equity_daily_bar_reader.trading_calendar
finder = bundle_data.asset_finder
data_portal = DataPortal(
finder,
calendar,
calendar.first_session,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader
)
return data_portal, calendar
def test_bundle(self):
with open(test_resource_path(
'quandl_samples',
'QUANDL_ARCHIVE.zip'), 'rb') as quandl_response:
self.responses.add(
self.responses.GET,
'https://file_url.mock.quandl',
body=quandl_response.read(),
content_type='application/zip',
status=200,
)
url_map = {
format_metadata_url(self.api_key): test_resource_path(
'quandl_samples',
'metadata.csv.gz',
)
}
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
with patch_read_csv(url_map):
ingest('quandl', environ=environ)
bundle = load('quandl', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.start_date, 'bfill')],
sessions[sessions.get_loc(self.end_date, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder,
)
assert_equal(actual, expected_pricing, array_decimal=2)
adjs_for_cols = bundle.adjustment_reader.load_pricing_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjs_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def test_bundle(self):
with open(test_resource_path(
'quandl_samples',
'QUANDL_ARCHIVE.zip'), 'rb') as quandl_response:
self.responses.add(
self.responses.GET,
'https://file_url.mock.quandl',
body=quandl_response.read(),
content_type='application/zip',
status=200,
)
url_map = {
format_metadata_url(self.api_key): test_resource_path(
'quandl_samples',
'metadata.csv.gz',
)
}
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
with patch_read_csv(url_map):
ingest('quandl', environ=environ)
bundle = load('quandl', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.start_date, 'bfill')],
sessions[sessions.get_loc(self.end_date, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder,
)
assert_equal(actual, expected_pricing, array_decimal=2)
adjs_for_cols = bundle.adjustment_reader.load_pricing_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjs_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def test_bundle(self):
with open(
join(TEST_RESOURCE_PATH, "quandl_samples",
"QUANDL_ARCHIVE.zip"),
"rb",
) as quandl_response:
self.responses.add(
self.responses.GET,
"https://file_url.mock.quandl",
body=quandl_response.read(),
content_type="application/zip",
status=200,
)
url_map = {
format_metadata_url(self.api_key):
join(
TEST_RESOURCE_PATH,
"quandl_samples",
"metadata.csv.gz",
)
}
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
"ZIPLINE_ROOT": zipline_root,
"QUANDL_API_KEY": self.api_key,
}
with patch_read_csv(url_map):
ingest("quandl", environ=environ)
bundle = load("quandl", environ=environ)
sids = 0, 1, 2, 3
assert set(bundle.asset_finder.sids) == set(sids)
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.start_date, "bfill")],
sessions[sessions.get_loc(self.end_date, "ffill")],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder, )
np.testing.assert_array_almost_equal(actual,
expected_pricing,
decimal=2)
adjs_for_cols = bundle.adjustment_reader.load_pricing_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns, adjs_for_cols,
expected_adjustments):
assert adjustments == expected, column
def test_mean_reversion_5day_sector_neutral_smoothed(fn):
column_name = 'Mean_Reversion_5Day_Sector_Neutral_Smoothed'
start_date_str = '2015-01-05'
end_date_str = '2015-01-07'
# Build engine
trading_calendar = get_calendar('NYSE')
bundle_data = bundles.load(project_helper.EOD_BUNDLE_NAME)
engine = project_helper.build_pipeline_engine(bundle_data,
trading_calendar)
# Build pipeline
universe_window_length = 2
universe_asset_count = 4
universe = AverageDollarVolume(
window_length=universe_window_length).top(universe_asset_count)
pipeline = Pipeline(screen=universe)
run_pipeline_args = {
'pipeline': pipeline,
'start_date': pd.Timestamp(start_date_str, tz='utc'),
'end_date': pd.Timestamp(end_date_str, tz='utc')
}
fn_inputs = {
'window_length': 3,
'universe': universe,
'sector': project_helper.Sector()
}
fn_correct_outputs = OrderedDict([
('pipline_out',
pd.DataFrame([
0.44721360, 1.34164079, -1.34164079, -0.44721360, 1.34164079,
0.44721360, -1.34164079, -0.44721360, 0.44721360, 1.34164079,
-1.34164079, -0.44721360
],
engine.run_pipeline(**run_pipeline_args).index,
[column_name]))
])
print('Running Integration Test on pipeline:')
print('> start_dat = pd.Timestamp(\'{}\', tz=\'utc\')'.format(
start_date_str))
print('> end_date = pd.Timestamp(\'{}\', tz=\'utc\')'.format(end_date_str))
print('> universe = AverageDollarVolume(window_length={}).top({})'.format(
universe_window_length, universe_asset_count))
print('> factor = {}('.format(fn.__name__))
print(' window_length={},'.format(fn_inputs['window_length']))
print(' universe=universe,')
print(' sector=project_helper.Sector())')
print('> pipeline.add(factor, \'{}\')'.format(column_name))
print('> engine.run_pipeline(pipeline, start_dat, end_date)')
print('')
pipeline.add(fn(**fn_inputs), column_name)
assert_output(engine.run_pipeline,
run_pipeline_args,
fn_correct_outputs,
check_parameter_changes=False)
def test_bundle(self):
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
# custom bundles need to be registered before use or they will not
# be recognized
register(
'ZacksQuandl',
from_zacks_dump(
test_resource_path('zacks_samples', 'fictitious.csv')))
ingest('ZacksQuandl', environ=environ)
# load bundle now that it has been ingested
bundle = load('ZacksQuandl', environ=environ)
sids = 0, 1, 2
# check sids match
assert_equal(set(bundle.asset_finder.sids), set(sids))
# check asset_{start, end} is the same as {start, end}_date
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
# get daily OHLCV data from bundle
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, 'bfill')],
sessions[sessions.get_loc(self.asset_end, 'ffill')],
sids,
)
# get expected data from csv
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder, )
# check OHLCV data matches
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def get_tickers_from_bundle(bundle_name):
"""Gets a list of tickers from a given bundle"""
bundle_data = bundles.load(bundle_name, os.environ, None)
# get a list of all sids
lifetimes = bundle_data.asset_finder._compute_asset_lifetimes("US")
all_sids = lifetimes.sid
# retreive all assets in the bundle
all_assets = bundle_data.asset_finder.retrieve_all(all_sids)
# return only tickers
return dict(map(lambda x: (x.symbol, x.sid), all_assets))
def _trading_calendar(bundle=DEFAULT_BUNDLE):
"""Create a TradingCalendar for the given bundle.
Parameters
----------
bundle : str
The name of the bundle to create a pipeline engine for.
Returns
-------
calendar : zipline.utils.calendars.TradingCalendar
The trading calendar for the bundle.
"""
bundle_data = bundles.load(bundle)
return bundle_data.equity_daily_bar_reader.trading_calendar
def _trading_calendar(bundle=DEFAULT_BUNDLE):
"""Create a TradingCalendar for the given bundle.
Parameters
----------
bundle : str
The name of the bundle to create a pipeline engine for.
Returns
-------
calendar : zipline.utils.calendars.TradingCalendar
The trading calendar for the bundle.
"""
bundle_data = bundles.load(bundle)
return bundle_data.equity_daily_bar_reader.trading_calendar
def test_mean_reversion_5day_sector_neutral_smoothed(fn):
column_name = 'Mean_Reversion_5Day_Sector_Neutral_Smoothed'
start_date_str = '2015-01-05'
end_date_str = '2015-01-07'
# Build engine
trading_calendar = get_calendar('NYSE')
bundle_data = bundles.load(project_helper.EOD_BUNDLE_NAME)
engine = project_helper.build_pipeline_engine(bundle_data, trading_calendar)
# Build pipeline
universe_window_length = 2
universe_asset_count = 4
universe = AverageDollarVolume(window_length=universe_window_length).top(universe_asset_count)
pipeline = Pipeline(screen=universe)
run_pipeline_args = {
'pipeline': pipeline,
'start_date': pd.Timestamp(start_date_str, tz='utc'),
'end_date': pd.Timestamp(end_date_str, tz='utc')}
fn_inputs = {
'window_length': 3,
'universe': universe,
'sector': project_helper.Sector()}
fn_correct_outputs = OrderedDict([
(
'pipline_out', pd.DataFrame(
[0.44721360, 1.34164079, -1.34164079, -0.44721360,
1.34164079, 0.44721360, -1.34164079, -0.44721360,
0.44721360, 1.34164079, -1.34164079, -0.44721360],
engine.run_pipeline(**run_pipeline_args).index,
[column_name]))])
print('Running Integration Test on pipeline:')
print('> start_dat = pd.Timestamp(\'{}\', tz=\'utc\')'.format(start_date_str))
print('> end_date = pd.Timestamp(\'{}\', tz=\'utc\')'.format(end_date_str))
print('> universe = AverageDollarVolume(window_length={}).top({})'.format(
universe_window_length, universe_asset_count))
print('> factor = {}('.format(fn.__name__))
print(' window_length={},'.format(fn_inputs['window_length']))
print(' universe=universe,')
print(' sector=project_helper.Sector())')
print('> pipeline.add(factor, \'{}\')'.format(column_name))
print('> engine.run_pipeline(pipeline, start_dat, end_date)')
print('')
pipeline.add(fn(**fn_inputs), column_name)
assert_output(engine.run_pipeline, run_pipeline_args, fn_correct_outputs, check_parameter_changes=False)
def _pipeline_engine_and_calendar_for_bundle(bundle):
"""Create a pipeline engine for the given bundle.
Parameters
----------
bundle : str
The name of the bundle to create a pipeline engine for.
Returns
-------
engine : zipline.pipleine.engine.SimplePipelineEngine
The pipeline engine which can run pipelines against the bundle.
calendar : zipline.utils.calendars.TradingCalendar
The trading calendar for the bundle.
"""
bundle_data = bundles.load(bundle)
if bundle == 'dwy':
pipeline_loader = EquityPricingLoader.without_fx(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
else:
# 分钟级别数据
pipeline_loader = EquityPricingLoader.without_fx(
bundle_data.equity_minute_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column.unspecialize() in EquityPricing.columns:
return pipeline_loader
# elif column in global_loader:
# return global_loader
else:
return global_loader
raise ValueError("%s is NOT registered in `PipelineLoader`." % column)
calendar = bundle_data.equity_daily_bar_reader.trading_calendar
return (
SimplePipelineEngine(
choose_loader,
bundle_data.asset_finder,
),
calendar,
)
def _asset_finder(bundle=DEFAULT_BUNDLE):
"""Create a AssetFinder for the given bundle.
Parameters
----------
bundle : str
The name of the bundle to create a pipeline engine for.
Returns
-------
engine : zipline.pipleine.engine.SimplePipelineEngine
The pipeline engine which can run pipelines against the bundle.
calendar : zipline.utils.calendars.TradingCalendar
The trading calendar for the bundle.
"""
bundle_data = bundles.load(bundle)
return bundle_data.asset_finder
def _asset_finder(bundle=DEFAULT_BUNDLE):
"""Create a AssetFinder for the given bundle.
Parameters
----------
bundle : str
The name of the bundle to create a pipeline engine for.
Returns
-------
engine : zipline.pipleine.engine.SimplePipelineEngine
The pipeline engine which can run pipelines against the bundle.
calendar : zipline.utils.calendars.TradingCalendar
The trading calendar for the bundle.
"""
bundle_data = bundles.load(bundle)
return bundle_data.asset_finder
def build_backtest_environment(star_date, end_date,capital_base=100000):
trading_calendar = get_calendar("NYSE")
sim_params = create_simulation_parameters(capital_base=capital_base,
data_frequency='daily',
trading_calendar=trading_calendar,
start=pd.Timestamp(pd.to_datetime(star_date)).tz_localize('US/Eastern'),
end=pd.Timestamp(pd.to_datetime(end_date)).tz_localize('US/Eastern')
)
bundle = bundles.load('quandl')
prefix, connstr = re.split(r'sqlite:///', str(bundle.asset_finder.engine.url), maxsplit=1, )
env = TradingEnvironment(asset_db_path=connstr, environ=os.environ)
data = DataPortal(
env.asset_finder, trading_calendar,
first_trading_day=bundle.equity_minute_bar_reader.first_trading_day,
equity_minute_reader=bundle.equity_minute_bar_reader,
equity_daily_reader=bundle.equity_daily_bar_reader,
adjustment_reader=bundle.adjustment_reader,
)
return data,env,bundle,sim_params
def get_window_price(end_date):
bundle_name = "custom-csv-bundle"
window = 30 # 窗口大小
bundle_data = bundles.load(bundle_name)
data_por = DataPortal(bundle_data.asset_finder,
get_calendar(calendar_name),
bundle_data.equity_daily_bar_reader.first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader)
sym = data_por.asset_finder.lookup_symbol(train_symbol_str, end_date)
data = data_por.get_history_window(assets=[sym],
end_dt=end_date,
bar_count=window,
frequency='1d',
data_frequency='daily',
field="close")
close = data.iloc[:, 0].values
return close
def test_bundle(self):
environ = {
"CSVDIR": join(
TEST_RESOURCE_PATH,
"csvdir_samples",
"csvdir",
),
}
ingest("csvdir", environ=environ)
bundle = load("csvdir", environ=environ)
sids = 0, 1, 2, 3
assert set(bundle.asset_finder.sids) == set(sids)
for equity in bundle.asset_finder.retrieve_all(sids):
assert equity.start_date == self.asset_start, equity
assert equity.end_date == self.asset_end, equity
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, "bfill")],
sessions[sessions.get_loc(self.asset_end, "ffill")],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder, )
np.testing.assert_array_almost_equal(actual,
expected_pricing,
decimal=2)
adjs_for_cols = bundle.adjustment_reader.load_pricing_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
assert [sorted(adj.keys())
for adj in adjs_for_cols] == expected_adjustments
def _pipeline_engine_and_calendar_for_bundle(bundle):
"""Create a pipeline engine for the given bundle.
Parameters
----------
bundle : str
The name of the bundle to create a pipeline engine for.
Returns
-------
engine : zipline.pipleine.engine.SimplePipelineEngine
The pipeline engine which can run pipelines against the bundle.
calendar : zipline.utils.calendars.TradingCalendar
The trading calendar for the bundle.
"""
bundle_data = bundles.load(bundle)
pipeline_loader = EquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column.unspecialize() in EquityPricing.columns:
return pipeline_loader
elif column in global_loader:
return global_loader
raise ValueError("%s is NOT registered in `PipelineLoader`." % column)
calendar = bundle_data.equity_daily_bar_reader.trading_calendar
return (
SimplePipelineEngine(
choose_loader,
bundle_data.asset_finder,
default_domain=CN_EQUITIES,
),
calendar,
)
def _pipeline_engine_and_calendar_for_bundle(bundle):
"""Create a pipeline engine for the given bundle.
Parameters
----------
bundle : str
The name of the bundle to create a pipeline engine for.
Returns
-------
engine : zipline.pipleine.engine.SimplePipelineEngine
The pipeline engine which can run pipelines against the bundle.
calendar : zipline.utils.calendars.TradingCalendar
The trading calendar for the bundle.
"""
bundle_data = bundles.load(bundle)
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError(
'No PipelineLoader registered for column %s.' % column
)
calendar = bundle_data.equity_daily_bar_reader.trading_calendar
return (
SimplePipelineEngine(
choose_loader,
calendar.all_sessions,
bundle_data.asset_finder,
),
calendar,
)
def _run(handle_data,
initialize,
before_trading_start,
analyze,
algofile,
algotext,
defines,
data_frequency,
capital_base,
data,
bundle,
bundle_timestamp,
start,
end,
output,
trading_calendar,
print_algo,
metrics_set,
local_namespace,
environ):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if trading_calendar is None:
trading_calendar = get_calendar('NYSE')
if trading_calendar.session_distance(start, end) < 1:
raise _RunAlgoError(
'There are no trading days between %s and %s' % (
start.date(),
end.date(),
),
)
if bundle is not None:
bundle_data = bundles.load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url),
)
env = TradingEnvironment(asset_db_path=connstr, environ=environ)
first_trading_day =\
bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
env.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError(
"No PipelineLoader registered for column %s." % column
)
else:
env = TradingEnvironment(environ=environ)
choose_loader = None
if isinstance(metrics_set, six.string_types):
try:
metrics_set = metrics.load(metrics_set)
except ValueError as e:
raise _RunAlgoError(str(e))
perf = TradingAlgorithm(
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
trading_calendar=trading_calendar,
sim_params=create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
trading_calendar=trading_calendar,
),
metrics_set=metrics_set,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}
).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
def set_bundle_data(bundle_name='alpaca_api'):
global BUNDLE_DATA, PRICING_LOADER
BUNDLE_DATA = bundles.load(bundle_name)
PRICING_LOADER = USEquityPricingLoader.without_fx(
BUNDLE_DATA.equity_daily_bar_reader, BUNDLE_DATA.adjustment_reader)
def test_bundle(self):
url_map = merge(
{
format_wiki_url(
self.api_key,
symbol,
self.start_date,
self.end_date,
): test_resource_path('quandl_samples', symbol + '.csv.gz')
for symbol in self.symbols
},
{
format_metadata_url(self.api_key, n): test_resource_path(
'quandl_samples',
'metadata-%d.csv.gz' % n,
)
for n in (1, 2)
},
)
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
with patch_read_csv(url_map, strict=True):
ingest('quandl', environ=environ)
bundle = load('quandl', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, 'bfill')],
sessions[sessions.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder,
)
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
algotext, defines, data_frequency, capital_base, data, bundle,
bundle_timestamp, start, end, output, trading_calendar, print_algo,
metrics_set, local_namespace, environ, bm_symbol):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign, )
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' %
(name, e), )
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if trading_calendar is None:
trading_calendar = get_calendar('SZSH')
if trading_calendar.session_distance(start, end) < 1:
raise _RunAlgoError(
'There are no trading days between %s and %s' % (
start.date(),
end.date(),
), )
if bundle is not None:
bundle_data = bundles.load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url), )
env = TradingEnvironment(
asset_db_path=connstr,
trading_calendar=trading_calendar,
# 构造使用字符串格式
exchange_tz=trading_calendar.tz.zone,
bm_symbol=bm_symbol,
environ=environ)
first_trading_day = bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
env.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
# # 简单处理
elif type(column) == BoundColumn:
return global_loader
raise ValueError("No PipelineLoader registered for column %s." %
column)
else:
env = TradingEnvironment(environ=environ)
choose_loader = None
if isinstance(metrics_set, six.string_types):
try:
metrics_set = metrics.load(metrics_set)
except ValueError as e:
raise _RunAlgoError(str(e))
perf = TradingAlgorithm(
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
trading_calendar=trading_calendar,
sim_params=create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
trading_calendar=trading_calendar,
),
metrics_set=metrics_set,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
def _run(handle_data,
initialize,
before_trading_start,
analyze,
algofile,
algotext,
defines,
data_frequency,
capital_base,
bundle,
bundle_timestamp,
start,
end,
output,
trading_calendar,
print_algo,
metrics_set,
local_namespace,
environ,
blotter,
benchmark_returns):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if benchmark_returns is None:
benchmark_returns, _ = load_market_data(environ=environ)
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if trading_calendar is None:
trading_calendar = get_calendar('XSHG')
# date parameter validation
if trading_calendar.session_distance(start, end) < 1:
raise _RunAlgoError(
'There are no trading days between %s and %s' % (
start.date(),
end.date(),
),
)
bundle_data = bundles.load(
bundle,
environ,
bundle_timestamp,
)
first_trading_day = \
bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
bundle_data.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
pipeline_loader = CNEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in CNEquityPricing.columns:
return pipeline_loader
# # 简单处理
elif type(column) == BoundColumn:
# # 使用实例才能避免KeyError
return global_loader
raise ValueError(
"No PipelineLoader registered for column %s." % column
)
if isinstance(metrics_set, six.string_types):
try:
metrics_set = metrics.load(metrics_set)
except ValueError as e:
raise _RunAlgoError(str(e))
if isinstance(blotter, six.string_types):
try:
blotter = load(Blotter, blotter)
except ValueError as e:
raise _RunAlgoError(str(e))
perf = TradingAlgorithm(
namespace=namespace,
data_portal=data,
get_pipeline_loader=choose_loader,
trading_calendar=trading_calendar,
sim_params=SimulationParameters(
start_session=start,
end_session=end,
trading_calendar=trading_calendar,
capital_base=capital_base,
data_frequency=data_frequency,
),
metrics_set=metrics_set,
blotter=blotter,
benchmark_returns=benchmark_returns,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}
).run()
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
def algo_args(year=2017,
from_date=None,
to_date=None,
capital_base=float('1.0e7'),
num_days=None,
metrics_set=None):
# Constant inputs
benchmark_sid = 3623 # '000300.SH'
bundle = "cndaily"
calendar = 'XSHG'
# With trading calendar
trading_calendar = get_calendar(calendar)
# With simulation parameters
if from_date:
from_date = pd.Timestamp(from_date, tz='utc')
if to_date:
to_date = pd.Timestamp(to_date, tz='utc')
sim_params = create_simulation_parameters(
year=year,
start=from_date,
end=to_date,
capital_base=capital_base,
num_days=num_days,
trading_calendar=trading_calendar
)
# With data portal
bundle_data = load(bundle)
data_portal = DataPortal(
bundle_data.asset_finder,
trading_calendar,
trading_calendar.first_session,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader
)
# With pipelineloader
pipeline_loader = EquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader
)
def choose_loader(column):
if column.unspecialize() in EquityPricing.columns:
return pipeline_loader
elif column in global_loader:
return global_loader
raise ValueError("%s is NOT registered in `PipelineLoader`." % column)
return {
'sim_params': sim_params,
'data_portal': data_portal,
'benchmark_sid': benchmark_sid,
'metrics_set': metrics_set,
'get_pipeline_loader': choose_loader,
}
def get_assets(ticker_count):
bundle = bundles.load('eod-quotemedia')
return bundle.asset_finder.retrieve_all(bundle.asset_finder.sids[:ticker_count])
def run_pipeline(pipeline, start_date, end_date=None, bundle=None):
"""
Compute values for pipeline from start_date to end_date, using the specified
bundle or the default bundle.
Parameters
----------
pipeline : Pipeline, required
The pipeline to run.
start_date : str (YYYY-MM-DD), required
First date on which the pipeline should run. If start_date is not a trading
day, the pipeline will start on the first trading day after start_date.
end_date : str (YYYY-MM-DD), optional
Last date on which the pipeline should run. If end_date is not a trading
day, the pipeline will end on the first trading day after end_date.
Defaults to today.
bundle : str, optional
the bundle code. If omitted, the default bundle will be used (and must be set).
Returns
-------
result : pd.DataFrame
A frame of computed results. The result columns correspond to the entries
of pipeline.columns, which should be a dictionary mapping strings to instances
of zipline.pipeline.term.Term. For each date between start_date and end_date,
result will contain a row for each asset that passed pipeline.screen. A screen
of None indicates that a row should be returned for each asset that existed each
day.
Examples
--------
Get a pipeline of 1-year returns:
>>> from zipline.pipeline.factors import Returns
>>> pipeline = Pipeline( # doctest: +SKIP
columns={
'1Y': Returns(window_length=252),
})
>>> factor = run_pipeline(pipeline, '2018-01-01', '2019-02-01', bundle="usstock-1min") # doctest: +SKIP
"""
if not bundle:
bundle = get_default_bundle()
if not bundle:
raise ValidationError("you must specify a bundle or set a default bundle")
bundle = bundle["default_bundle"]
load_extensions(code=bundle)
bundle_data = bundles.load(
bundle,
os.environ,
pd.Timestamp.utcnow(),
)
calendar_name = bundles.bundles[bundle].calendar_name
trading_calendar = get_calendar(calendar_name)
start_date = pd.Timestamp(start_date)
if start_date.tz:
start_date = start_date.tz_convert("UTC")
else:
start_date = start_date.tz_localize("UTC")
if end_date:
end_date = pd.Timestamp(end_date)
else:
end_date = pd.Timestamp.now().normalize()
if end_date.tz:
end_date = end_date.tz_convert("UTC")
else:
end_date = end_date.tz_localize("UTC")
first_session = max(bundles.bundles[bundle].start_session, trading_calendar.first_session)
if start_date < first_session:
raise ValidationError(
f"start_date cannot be earlier than {first_session.date().isoformat()} for this bundle")
# Roll-forward start_date to valid session
for i in range(100):
if trading_calendar.is_session(start_date):
break
start_date += pd.Timedelta(days=1)
else:
raise ValidationError(f"start_date is not in {calendar_name} calendar")
# Roll-forward end_date to valid session
for i in range(100):
if trading_calendar.is_session(end_date):
break
end_date += pd.Timedelta(days=1)
else:
raise ValidationError("end_date is not in calendar")
if (
end_date < start_date):
raise ValidationError("end_date cannot be earlier than start_date")
default_pipeline_loader = EquityPricingLoader.without_fx(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
asset_finder = asset_finder_cache.get(bundle, bundle_data.asset_finder)
asset_finder_cache[bundle] = asset_finder
pipeline_loader = QuantRocketPipelineLoaderRouter(
asset_db_conn=asset_finder.engine,
calendar=trading_calendar,
default_loader=default_pipeline_loader,
default_loader_columns=EquityPricing.columns
)
calendar_domain = domain.get_domain_from_calendar(trading_calendar)
engine = SimplePipelineEngine(
pipeline_loader,
asset_finder,
calendar_domain)
return engine.run_pipeline(pipeline, start_date, end_date)
def test_bundle(self):
url_map = merge(
{
format_wiki_url(
self.api_key,
symbol,
self.start_date,
self.end_date,
): test_resource_path('quandl_samples', symbol + '.csv.gz')
for symbol in self.symbols
},
{
format_metadata_url(self.api_key, n): test_resource_path(
'quandl_samples',
'metadata-%d.csv.gz' % n,
)
for n in (1, 2)
},
)
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
with patch_read_csv(url_map, strict=True):
ingest('quandl', environ=environ)
bundle = load('quandl', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
cal = self.calendar
actual = bundle.daily_bar_reader.load_raw_arrays(
self.columns,
cal[cal.get_loc(self.asset_start, 'bfill')],
cal[cal.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder,
)
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
cal,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
import os
from zipline.utils.run_algo import load_extensions
from zipline.api import get_datetime
load_extensions(
default=True,
extensions=[],
strict=True,
environ=os.environ,
)
from datetime import date, timedelta
from zipline.data import bundles
import threading, Queue
bundle = bundles.load('ftx')
bundle.asset_finder.retrieve_all(bundle.asset_finder.sids)
# imports
from zipline.api import order, order_value, symbol, record, set_benchmark, get_datetime
from zipline.protocol import Portfolio
# parameters
import pandas as pd
import os
lts = ['ADABULL/USD', 'ALGOBULL/USD', 'ALTBULL/USD', 'ATOMBULL/USD', 'BALBULL/USD', 'BCHBULL/USD', 'BNBBULL/USD', 'BSVBULL/USD', 'BTMXBULL/USD', 'BULL/USD', 'COMPBULL/USD', 'DEFIBULL/USD', 'DOGEBULL/USD', 'DRGNBULL/USD', 'EOSBULL/USD', 'ETCBULL/USD', 'ETHBULL/USD', 'EXCHBULL/USD', 'HTBULL/USD', 'KNCBULL/USD', 'LEOBULL/USD', 'LINKBULL/USD', 'LTCBULL/USD', 'MATICBULL/USD', 'MIDBULL/USD', 'OKBBULL/USD', 'PAXGBULL/USD', 'PRIVBULL/USD', 'THETABULL/USD', 'TOMOBULL/USD', 'TRXBULL/USD', 'TRYBBULL/USD', 'XAUTBULL/USD', 'XRPBULL/USD', 'XTZBULL/USD']
import datetime
assets = []
has_ordereds = {}
prices = {}
def _run(handle_data,
initialize,
before_trading_start,
analyze,
algofile,
algotext,
defines,
data_frequency,
capital_base,
bundle,
bundle_timestamp,
custom_data_portal,
start,
end,
output,
trading_calendar,
print_algo,
metrics_set,
local_namespace,
environ,
blotter,
benchmark_returns):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if benchmark_returns is None:
benchmark_returns, _ = load_market_data(environ=environ)
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if trading_calendar is None:
trading_calendar = get_calendar('XNYS')
# date parameter validation
if trading_calendar.session_distance(start, end) < 1:
raise _RunAlgoError(
'There are no trading days between %s and %s' % (
start.date(),
end.date(),
),
)
bundle_data = bundles.load(
bundle,
environ,
bundle_timestamp,
)
# TODO: Fix this for the custom DataPortal case.
first_trading_day = \
bundle_data.equity_minute_bar_reader.first_trading_day
if custom_data_portal is None:
data = DataPortal(
bundle_data.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
else:
data = custom_data_portal
# TODO: Fix this for the custom DataPortal case.
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError(
"No PipelineLoader registered for column %s." % column
)
if isinstance(metrics_set, six.string_types):
try:
metrics_set = metrics.load(metrics_set)
except ValueError as e:
raise _RunAlgoError(str(e))
if isinstance(blotter, six.string_types):
try:
blotter = load(Blotter, blotter)
except ValueError as e:
raise _RunAlgoError(str(e))
perf = TradingAlgorithm(
namespace=namespace,
data_portal=data,
get_pipeline_loader=choose_loader,
trading_calendar=trading_calendar,
sim_params=SimulationParameters(
start_session=start,
end_session=end,
trading_calendar=trading_calendar,
capital_base=capital_base,
data_frequency=data_frequency,
),
metrics_set=metrics_set,
blotter=blotter,
benchmark_returns=benchmark_returns,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}
).run()
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
def get_assets(ticker_count):
bundle = bundles.load('eod-quotemedia')
return bundle.asset_finder.retrieve_all(
bundle.asset_finder.sids[:ticker_count])
def test_bundle(self):
def get_symbol_from_url(url):
params = parse_qs(urlparse(url).query)
symbol, = params['s']
return symbol
def pricing_callback(request):
headers = {
'content-encoding': 'gzip',
'content-type': 'text/csv',
}
path = test_resource_path(
'yahoo_samples',
get_symbol_from_url(request.url) + '.csv.gz',
)
with open(path, 'rb') as f:
return (
200,
headers,
f.read(),
)
for _ in range(3):
self.responses.add_callback(
self.responses.GET,
'http://ichart.finance.yahoo.com/table.csv',
pricing_callback,
)
def adjustments_callback(request):
path = test_resource_path(
'yahoo_samples',
get_symbol_from_url(request.url) + '.adjustments.gz',
)
return 200, {}, read_compressed(path)
for _ in range(3):
self.responses.add_callback(
self.responses.GET,
'http://ichart.finance.yahoo.com/x',
adjustments_callback,
)
cal = self.calendar
self.register(
'bundle',
yahoo_equities(self.symbols),
calendar=cal,
)
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
}
self.ingest('bundle', environ=environ)
bundle = load('bundle', environ=environ)
sids = 0, 1, 2
equities = bundle.asset_finder.retrieve_all(sids)
for equity, expected_symbol in zip(equities, self.symbols):
assert_equal(equity.symbol, expected_symbol)
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
cal[cal.get_loc(self.asset_start, 'bfill')],
cal[cal.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data()
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
cal,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
from zipline.utils.calendars import get_calendar
from zipline.assets._assets import Future
from zipline.utils.run_algo import load_extensions
# Load extensions.py; this allows you access to custom bundles
load_extensions(
default=True,
extensions=[],
strict=True,
environ=os.environ,
)
# Set-Up Pricing Data Access
trading_calendar = get_calendar('NYSE')
bundle = 'futures'
bundle_data = bundles.load(bundle)
data = DataPortal(
bundle_data.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=bundle_data.equity_daily_bar_reader.first_trading_day,
equity_minute_reader=None,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
future_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
future = bundle_data.asset_finder.lookup_future_symbol
continuous_future = bundle_data.asset_finder.create_continuous_future
history = data.get_history_window
def sid(sid, bundle=None):
"""
Return an Asset object for the specified sid in the specified bundle
(or default bundle).
Parameters
----------
sid : str, required
The sid to retrieve.
bundle : str, optional
the bundle code. If omitted, the default bundle will be used (and must be set).
Returns
-------
asset : zipline.assets.Asset
Notes
-----
Each asset is specific to the bundle from which it came. An
Asset object for AAPL from bundle A cannot be used to retrieve
AAPL data from bundle B, even if AAPL data is present in bundle
B.
Examples
--------
Get the asset object for AAPL:
>>> aapl = sid("FIBBG000B9XRY4", bundle="usstock-1min")
"""
if not bundle:
bundle = get_default_bundle()
if not bundle:
raise ValidationError(
"you must specify a bundle or set a default bundle")
bundle = bundle["default_bundle"]
load_extensions(code=bundle)
bundle_data = bundles.load(
bundle,
os.environ,
pd.Timestamp.utcnow(),
)
asset_finder = asset_finder_cache.get(bundle, bundle_data.asset_finder)
asset_finder_cache[bundle] = asset_finder
zipline_sid = asset_finder.engine.execute(
"""
SELECT
sid
FROM
equities
WHERE
real_sid = ?
UNION
SELECT
sid
FROM
futures_contracts
WHERE
real_sid = ?
""", (sid, sid)).scalar()
if not zipline_sid:
raise ValidationError(f"No such sid {sid} in {bundle} bundle")
asset = asset_finder.retrieve_asset(zipline_sid)
return asset
def initialize(context):
context.set_benchmark(None)
context.i = 1
context.assets = list(
map(lambda x: symbol(x), high_cap_company.Symbol.values))
print(context.assets, len(context.assets))
context.model_fee = 5e-3
context.set_commission(commission.PerShare(cost=0.005, min_trade_cost=1.0))
context.set_slippage(slippage.VolumeShareSlippage())
context.bootstrap_sequence_length = 300
context.max_sequence_length = 60
context.tb_log_dir = './log/%s' % back_test_name
context.model_update_time = 30
context.target_profit_multiplier = 1.1
context.model_summaries = None
bundle = bundles.load('quandl')
start_date_str = str(context.get_datetime().date())
initial_history_start_date = bundle.equity_daily_bar_reader.sessions[
bundle.equity_daily_bar_reader.sessions < start_date_str][(
-context.bootstrap_sequence_length - 1)]
initial_history_end_date = bundle.equity_daily_bar_reader.sessions[
bundle.equity_daily_bar_reader.sessions > start_date_str][0]
filterd_assets_index = (np.isnan(
np.sum(bundle.equity_daily_bar_reader.load_raw_arrays(
columns=['close'],
start_date=initial_history_start_date,
end_date=initial_history_end_date,
assets=context.assets),
axis=1)).flatten() == False)
context.assets = list(np.array(context.assets)[filterd_assets_index])
print(context.assets, len(context.assets))
remain_symbols = list(map(lambda x: x.symbol, context.assets))
if not os.path.exists('history_data'):
print('Start to download good history data')
history_data = {}
for s in remain_symbols:
print('downloading', s)
stock = quandl.get_table(
'WIKI/PRICES',
date={'gte': str(initial_history_start_date)},
ticker=s)
stock.index = stock.date
history_data[s] = stock
history_data = pd.Panel(history_data)
history_data.to_pickle('history_data')
context.history_data = generate_stock_features(history_data)
print('Done')
else:
print('history data exist')
history_data = pd.read_pickle('history_data')
context.history_data = generate_stock_features(history_data)
if not os.path.exists('index'):
print('downloading index data')
spy = quandl.get("CHRIS/CME_SP1", authtoken="CTq2aKvtCkPPgR4L_NFs")
gc = quandl.get("CHRIS/CME_GC1", authtoken="CTq2aKvtCkPPgR4L_NFs")
si = quandl.get("CHRIS/CME_SI1", authtoken="CTq2aKvtCkPPgR4L_NFs")
vix = pd.read_csv(
'http://www.cboe.com/publish/scheduledtask/mktdata/datahouse/vixcurrent.csv'
)
vix.columns = vix.iloc[0]
vix = vix[1:]
vix.index = pd.DatetimeIndex(vix.Date)
vix = vix.drop('Date', axis=1)
vix = vix.astype(np.float64)
vix.columns = ['Open', 'High', 'Low', 'Last']
index_data = pd.Panel({'vix': vix, 'gc': gc, 'si': si, 'spy': spy})
index_data.to_pickle('index')
index_data = index_data[:, str(initial_history_start_date):, :]
context.index_data = generate_index_features(index_data)
else:
print('index data exist')
index_data = pd.read_pickle('index')
index_data = index_data[:, str(initial_history_start_date):, :]
context.index_data = generate_index_features(
index_data)[:, context.history_data.major_axis[0]:, :]
if not os.path.exists('trading_content'):
sys.exit(1)
else:
news_vec = pd.read_csv('trading_content')
news_vec.index = news_vec.date
news_vec = news_vec.drop('date', axis=1)
news_vec = context.history_data[:, :,
'return_rate'].join(news_vec).drop(
context.history_data.items,
axis=1).fillna(0)
context.news_vec = news_vec
assert context.history_data.major_axis[0] == context.index_data.major_axis[
0]
feature_network_topology = {
'equity_network': {
'feature_map_number': len(context.assets),
'feature_number': context.history_data.shape[2],
'input_name': 'equity',
'dense': {
'n_units': [128, 64],
'act': [tf.nn.tanh] * 2,
},
'rnn': {
'n_units': [32, 1],
'act': [tf.nn.tanh, None],
'attention_length': 10
},
'keep_output': True
},
'index_network': {
'feature_map_number': len(context.index_data.items),
'feature_number': context.index_data.shape[2],
'input_name': 'index',
'dense': {
'n_units': [128, 64],
'act': [tf.nn.tanh] * 2,
},
'rnn': {
'n_units': [32, 16],
'act': [tf.nn.tanh, tf.nn.tanh],
'attention_length': 10
},
'keep_output': False
},
'weight_network': {
'feature_map_number': 1,
'feature_number': len(context.assets) + 1,
'input_name': 'weight',
'dense': {
'n_units': [32, 16],
'act': [tf.nn.tanh] * 2,
},
'rnn': {
'n_units': [16, 8],
'act': [tf.nn.tanh, tf.nn.tanh],
'attention_length': 10
},
'keep_output': False
},
'return_network': {
'feature_map_number': 1,
'feature_number': 1,
'input_name': 'return',
'dense': {
'n_units': [8, 4],
'act': [tf.nn.tanh] * 2,
},
'rnn': {
'n_units': [4, 2],
'act': [tf.nn.tanh, tf.nn.tanh],
'attention_length': 10
},
'keep_output': False
},
'news_network': {
'feature_map_number': 1,
'feature_number': 100,
'input_name': 'return',
'dense': {
'n_units': [128, 64],
'act': [tf.nn.tanh] * 2,
},
'rnn': {
'n_units': [32, 16],
'act': [tf.nn.tanh, tf.nn.tanh],
'attention_length': 10
},
'keep_output': False
}
}
context.model = DRL_Portfolio(
asset_number=len(context.assets),
feature_network_topology=feature_network_topology,
action_network_layers=[32, 16],
object_function='reward')
context.real_return = []
context.history_weight = []
context.model.init_model()
context.tensorboard = TensorBoard(log_dir=context.tb_log_dir,
session=context.model.get_session())
def test_ingest(self):
zipline_root = self.enter_instance_context(tmp_dir()).path
env = self.enter_instance_context(tmp_trading_env())
start = pd.Timestamp('2014-01-06', tz='utc')
end = pd.Timestamp('2014-01-10', tz='utc')
calendar = trading_days[trading_days.slice_indexer(start, end)]
minutes = env.minutes_for_days_in_range(calendar[0], calendar[-1])
outer_environ = {
'ZIPLINE_ROOT': zipline_root,
}
sids = tuple(range(3))
equities = make_simple_equity_info(
sids,
calendar[0],
calendar[-1],
)
daily_bar_data = make_bar_data(equities, calendar)
minute_bar_data = make_bar_data(equities, minutes)
first_split_ratio = 0.5
second_split_ratio = 0.1
splits = pd.DataFrame.from_records([
{
'effective_date': str_to_seconds('2014-01-08'),
'ratio': first_split_ratio,
'sid': 0,
},
{
'effective_date': str_to_seconds('2014-01-09'),
'ratio': second_split_ratio,
'sid': 1,
},
])
@self.register('bundle',
calendar=calendar,
opens=env.opens_in_range(calendar[0], calendar[-1]),
closes=env.closes_in_range(calendar[0], calendar[-1]))
def bundle_ingest(environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
cache,
show_progress):
assert_is(environ, outer_environ)
asset_db_writer.write(equities=equities)
minute_bar_writer.write(minute_bar_data)
daily_bar_writer.write(daily_bar_data)
adjustment_writer.write(splits=splits)
assert_is_instance(calendar, pd.DatetimeIndex)
assert_is_instance(cache, dataframe_cache)
assert_is_instance(show_progress, bool)
self.ingest('bundle', environ=outer_environ)
bundle = load('bundle', environ=outer_environ)
assert_equal(set(bundle.asset_finder.sids), set(sids))
columns = 'open', 'high', 'low', 'close', 'volume'
actual = bundle.minute_bar_reader.load_raw_arrays(
columns,
minutes[0],
minutes[-1],
sids,
)
for actual_column, colname in zip(actual, columns):
assert_equal(
actual_column,
expected_bar_values_2d(minutes, equities, colname),
msg=colname,
)
actual = bundle.daily_bar_reader.load_raw_arrays(
columns,
calendar[0],
calendar[-1],
sids,
)
for actual_column, colname in zip(actual, columns):
assert_equal(
actual_column,
expected_bar_values_2d(calendar, equities, colname),
msg=colname,
)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
columns,
calendar,
pd.Index(sids),
)
for column, adjustments in zip(columns, adjustments_for_cols[:-1]):
# iterate over all the adjustments but `volume`
assert_equal(
adjustments,
{
2: [Float64Multiply(
first_row=0,
last_row=2,
first_col=0,
last_col=0,
value=first_split_ratio,
)],
3: [Float64Multiply(
first_row=0,
last_row=3,
first_col=1,
last_col=1,
value=second_split_ratio,
)],
},
msg=column,
)
# check the volume, the value should be 1/ratio
assert_equal(
adjustments_for_cols[-1],
{
2: [Float64Multiply(
first_row=0,
last_row=2,
first_col=0,
last_col=0,
value=1 / first_split_ratio,
)],
3: [Float64Multiply(
first_row=0,
last_row=3,
first_col=1,
last_col=1,
value=1 / second_split_ratio,
)],
},
msg='volume',
)
