def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register('test', lambda *args: None)
cls.add_class_callback(partial(unregister, 'test'))
with tarfile.open(test_resource_path('example_data.tar.gz')) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath(
'example_data/expected_perf/%s' %
pd.__version__.replace('.', '-'), ),
serialization='pickle',
)
# We need to call gc.collect before tearing down our class because we
# have a cycle between TradingAlgorithm and AlgorithmSimulator which
# ultimately holds a reference to the pipeline engine passed to the
# tests here.
# This means that we're not guaranteed to have deleted our disk-backed
# resource readers (e.g. SQLiteAdjustmentReader) before trying to
# delete the tempdir, which causes failures on Windows because Windows
# doesn't allow you to delete a file if someone still has an open
# handle to that file.
# :(
cls.add_class_callback(gc.collect)
def make_pipeline_engine(symbols=['SPY', 'TLT'], bundle='etfs_bundle', calendar='NYSE'):
register(bundle, symbols)
bundle_data = load(bundle)
# Set up pipeline engine
# Loader for pricing
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def my_dispatcher(column):
return loaders[column]
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
return my_dispatcher(column)
trading_calendar = get_calendar(calendar)
engine = SimplePipelineEngine(
get_loader=choose_loader,
calendar=trading_calendar.all_sessions,
asset_finder=bundle_data.asset_finder,
)
assets = bundle_data.asset_finder.lookup_symbols(symbols, as_of_date=None)
return assets, engine
def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register("test", lambda *args: None)
cls.add_class_callback(partial(unregister, "test"))
with tarfile.open(test_resource_path("example_data.tar.gz")) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath("example_data/expected_perf/%s" % pd.__version__.replace(".", "-")),
serialization="pickle",
)
# We need to call gc.collect before tearing down our class because we
# have a cycle between TradingAlgorithm and AlgorithmSimulator which
# ultimately holds a reference to the pipeline engine passed to the
# tests here.
# This means that we're not guaranteed to have deleted our disk-backed
# resource readers (e.g. SQLiteAdjustmentReader) before trying to
# delete the tempdir, which causes failures on Windows because Windows
# doesn't allow you to delete a file if someone still has an open
# handle to that file.
# :(
cls.add_class_callback(gc.collect)
def main():
register_calendar('BITMEX', AlwaysOpenCalendar())
# The following is necessary because zipline's developer hard-coded NYSE
# everywhere in run_algo._run, *DOH*!!!
deregister_calendar('NYSE')
register_calendar_alias('NYSE', 'BITMEX', force=False)
register(
'bitmex',
create_bundle(
[
Pairs.symbol_btc,
#Pairs.symbol_ada,
#Pairs.symbol_bch,
#Pairs.symbol_eos,
#Pairs.symbol_eth,
#Pairs.symbol_ltc,
#Pairs.symbol_trx,
#Pairs.symbol_xrp,
],
pd.Timestamp('2017-11-05 00:00:00', tz='utc'),
pd.Timestamp('2018-12-18 23:59:59', tz='utc'),
),
calendar_name='BITMEX',
minutes_per_day=24*60
)
def load_data():
print('Loading data...')
from zipline.data.bundles import register
from estimize.zipline.data.bundles.yahoo import yahoo_bundle
tickers = {
'SPY',
}
register(
'yahoo',
yahoo_bundle(tickers),
)
bundles_module.ingest(
'yahoo',
os.environ,
pd.Timestamp.utcnow(),
[],
True,
)
bundles_module.ingest(
'quantopian-quandl',
os.environ,
pd.Timestamp.utcnow(),
[],
True,
)
def ingest(bundle, assets_version, show_progress):
if bundle == 'tdx':
register('tdx', tdx_bundle, 'SHSZ')
bundles_module.ingest(bundle,
os.environ,
pd.Timestamp.utcnow(),
assets_version,
show_progress,
)
def test_bundle(self):
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
# custom bundles need to be registered before use or they will not
# be recognized
register(
'ZacksQuandl',
from_zacks_dump(
test_resource_path('zacks_samples', 'fictitious.csv')))
ingest('ZacksQuandl', environ=environ)
# load bundle now that it has been ingested
bundle = load('ZacksQuandl', environ=environ)
sids = 0, 1, 2
# check sids match
assert_equal(set(bundle.asset_finder.sids), set(sids))
# check asset_{start, end} is the same as {start, end}_date
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
# get daily OHLCV data from bundle
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, 'bfill')],
sessions[sessions.get_loc(self.asset_end, 'ffill')],
sids,
)
# get expected data from csv
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder, )
# check OHLCV data matches
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def __register_bundle__(self):
from zipline.data.bundles import register
from zipline.data.bundles import ingest
register(
self.bundle_name, # name this whatever you like
self.tradea_bundle(self.symbol_list),
)
ingest(self.bundle_name)
def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register("test", lambda *args: None)
cls.add_class_callback(partial(unregister, "test"))
with tarfile.open(test_resource_path("example_data.tar.gz")) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath("example_data/expected_perf/%s" % pd.__version__.replace(".", "-")),
serialization="pickle",
)
def main(job_id, D):
print "job_id", job_id, " params:", D
equities1 = {}
register(
'my-db-bundle', # name this whatever you like
viadb(equities1),
calendar='SHSZ')
parsed = {}
parsed['initialize'] = None
parsed['handle_data'] = None
parsed['before_trading_start'] = None
parsed['analyze'] = None
parsed['algotext'] = None
parsed['defines'] = ()
parsed['capital_base'] = 1000000
parsed['data'] = None
parsed['bundle'] = 'my-db-bundle'
#parsed['bundle']='YAHOO'
#parsed['bundle_timestamp']=None
parsed['bundle_timestamp'] = pd.Timestamp.utcnow()
parsed['start'] = Timestamp('2017-03-01 13:30:00+0000', tz='UTC')
parsed['end'] = Timestamp('2017-06-01 13:30:00+0000', tz='UTC')
parsed['algofile'] = open(
'/data/kanghua/workshop/strategy/campaign/hyperparam/example-new/zipline_strategy.py'
)
parsed['data_frequency'] = 'daily'
parsed['print_algo'] = False
parsed['output'] = 'os.devnull'
parsed['local_namespace'] = None
parsed['environ'] = os.environ
parsed['bm_symbol'] = None
# Below what we expect spearmint to pass us
# parsed['algo_params']=[47,88.7,7.7]
# D={}
# D['timeperiod']=10
# D['nbdevup']=1.00
# D['nbdevdn']=1.00
parsed['algo_params'] = D
perf = _run(**parsed)
StartV = perf['portfolio_value'][0]
EndV = perf['portfolio_value'][-1]
# spearmint wants to minimize so return negative profit
OPTIM = (StartV - EndV)
return OPTIM
def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register('test', lambda *args: None)
cls.add_class_callback(partial(unregister, 'test'))
with tarfile.open(test_resource_path('example_data.tar.gz')) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath(
'example_data/expected_perf/%s' %
pd.__version__.replace('.', '-'), ),
serialization='pickle',
)
def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register('test', lambda *args: None)
cls.add_class_callback(partial(unregister, 'test'))
with tarfile.open(test_resource_path('example_data.tar.gz')) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath(
'example_data/expected_perf/%s' %
pd.__version__.replace('.', '-'),
),
serialization='pickle',
)
def main():
register_calendar('CN_FUTURES', CN_FUTURE_CALENDAR(), force=True)
deregister_calendar('NYSE')
register_calendar_alias('NYSE', 'CN_FUTURES', force=True)
register(
'CN_FUTURES',
create_bundle(
['IC', 'IH', 'IF', 'RB'],
pd.Timestamp('2017-09-20 01:31:00', tz='utc'),
pd.Timestamp('2018-10-31 01:31:00', tz='utc'),
),
calendar_name='CN_FUTURES',
minutes_per_day=4 * 60 + 90
)
def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register('test', lambda *args: None)
cls.add_class_callback(partial(unregister, 'test'))
with tarfile.open(test_resource_path('example_data.tar.gz')) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath(
'example_data/expected_perf/%s' %
pd.__version__.replace('.', '-'), ),
serialization='pickle',
)
market_data = ('SPY_benchmark.csv', 'treasury_curves.csv')
for data in market_data:
ensure_file(cls.tmpdir.getpath('example_data/root/data/' + data))
def ingest(bundle, assets, minute, assets_version, show_progress):
if bundle == 'tdx':
if assets:
if not os.path.exists(assets):
raise FileNotFoundError
df = pd.read_csv(assets,
names=['symbol', 'name'],
dtype=str,
encoding='utf8')
register('tdx', partial(tdx_bundle, df, minute), 'SHSZ')
else:
register('tdx', partial(tdx_bundle, None, minute), 'SHSZ')
bundles_module.ingest(
bundle,
os.environ,
pd.Timestamp.utcnow(),
assets_version,
show_progress,
)
def register_cn_bundle_from_yahoo(name, cache=True):
"""
register a new bundle of stocks from chinese market from yahoo
:param name: the name of bundle
:return: register result
"""
symbol_list = get_filtered_symbols(cache)
return register(
name,
yahoo_equities(dict(list(zip(symbol_list, symbol_list)))),
)
def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register("test", lambda *args: None)
cls.add_class_callback(partial(unregister, "test"))
with tarfile.open(test_resource_path("example_data.tar.gz")) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath(
"example_data/expected_perf/%s" %
pd.__version__.replace(".", "-"), ),
serialization="pickle",
)
cls.no_benchmark_expected_perf = {
example_name:
cls._no_benchmark_expectations_applied(expected_perf.copy())
for example_name, expected_perf in cls.expected_perf.items()
}
def init_class_fixtures(cls):
super().init_class_fixtures()
register('test', lambda *args: None)
cls.add_class_callback(partial(unregister, 'test'))
with tarfile.open(test_resource_path('example_data.tar.gz')) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath(
'example_data/expected_perf/%s' %
pd.__version__.replace('.', '-'), ),
serialization='pickle',
)
cls.no_benchmark_expected_perf = {
example_name:
cls._no_benchmark_expectations_applied(expected_perf.copy())
for example_name, expected_perf in cls.expected_perf.items()
}
def target_ingest(assets, ingest_minute=False):
import cn_stock_holidays.zipline.default_calendar
if assets:
if not os.path.exists(assets):
raise FileNotFoundError
df = pd.read_csv(assets,
names=['symbol', 'name'],
dtype=str,
encoding='utf8')
register('tdx', partial(tdx_bundle, df[:1], ingest_minute), 'SHSZ')
else:
df = pd.DataFrame({'symbol': ['000001'], 'name': ['平安银行']})
register('tdx', partial(tdx_bundle, df, ingest_minute), 'SHSZ')
bundles_module.ingest(
'tdx',
os.environ,
pd.Timestamp.utcnow(),
show_progress=True,
)
def register_tdx(assets=None,
minute=False,
start=None,
overwrite=False,
end=None):
calendar = get_calendar('SHSZ')
if not end:
now = pd.to_datetime('now')
if now.time() < pd.to_datetime('15:30').tz_localize(
'Asia/Shanghai').tz_convert("UTC").time():
end = now.date() - pd.Timedelta('24 H')
end = calendar.all_sessions[calendar.all_sessions.get_loc(
end, method='ffill')]
else:
end = now.date()
if start:
if not calendar.is_session(start):
start = calendar.all_sessions[searchsorted(calendar.all_sessions,
start)]
register('tdx', partial(tdx_bundle, assets, minute, overwrite), 'SHSZ',
start, end)
class YahooConfig(Config):
YAHOO_TICKERS = {
'SPY',
}
register(
'yahoo',
yahoo_bundle(YAHOO_TICKERS),
)
def __init__(self):
super(YahooConfig, self).__init__(bundle_name='yahoo')
def _setup_class(request, tmpdir_factory):
request.cls.tmp_path = tmpdir_factory.mktemp("tmp")
request.cls.tmpdir = str(request.cls.tmp_path)
register("test", lambda *args: None)
with tarfile.open(join(TEST_RESOURCE_PATH, "example_data.tar.gz")) as tar:
tar.extractall(request.cls.tmpdir)
request.cls.expected_perf_dirs = listdir(
join(
str(request.cls.tmp_path),
"example_data",
"expected_perf",
))
if PANDAS_VERSION not in request.cls.expected_perf_dirs:
warnings.warn(
"No data stored matches the current version of pandas. "
"Consider including the data using rebuild_example_data", )
yield
partial(unregister, "test")
def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register('test', lambda *args: None)
cls.add_class_callback(partial(unregister, 'test'))
with tarfile.open(test_resource_path('example_data.tar.gz')) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath(
'example_data/expected_perf/%s' %
pd.__version__.replace('.', '-'),
),
serialization='pickle',
)
market_data = ('SPY_benchmark.csv', 'treasury_curves.csv')
for data in market_data:
update_modified_time(
cls.tmpdir.getpath(
'example_data/root/data/' + data
)
)
# start_date += timedelta(days=1)
start_date = end_date - timedelta(days=365)
while not cal.is_session(start_date):
start_date -= timedelta(days=1)
initialize_client()
import time
start_time = time.time()
register(
'alpaca_api',
# api_to_bundle(interval=['1d', '1m']),
# api_to_bundle(interval=['1m']),
api_to_bundle(interval=['1d']),
calendar_name='NYSE',
start_session=start_date,
end_session=end_date)
assets_version = ((), )[0] # just a weird way to create an empty tuple
bundles_module.ingest(
"alpaca_api",
os.environ,
assets_versions=assets_version,
show_progress=True,
)
print(f"--- It took {timedelta(seconds=time.time() - start_time)} ---")
@property
def name(self):
return "bitxex"
@property
def tz(self):
return timezone('UTC')
@property
def open_time(self):
return time(0, 0)
@property
def close_time(self):
return time(23, 59)
@lazyval
def day(self):
return CustomBusinessDay(weekmask='Mon Tue Wed Thu Fri Sat Sun')
if __name__ == '__main__':
register_calendar('bitxex', BitmexCalendar())
register('bitxex',
bitmex(['XBTUSD']),
calendar_name='bitxex',
start_session=pd.Timestamp('2018-05-20', tz='utc'),
end_session=pd.Timestamp('2018-06-10', tz='utc'),
minutes_per_day=24 * 60)
'Predict Factor': predict.downsample('week_start'),
'longs': longs.downsample('week_start'),
'shorts': shorts.downsample('week_start'),
'predict_rank': predict_rank.downsample('week_start'),
}
pipe = Pipeline(
columns=pipe_final_columns,
screen=long_short_screen,
)
return pipe
############################################# bundle #############################################
equities1 = {}
register(
'my-db-bundle', # name this whatever you like
viadb(equities1),
calendar='SHSZ')
bundle = 'my-db-bundle'
bundle_timestamp = pd.Timestamp.utcnow()
environ = os.environ
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
#print prefix, connstr
from zipline.data.bundles import register
from zipline.data.bundles.google import google_equities, viacsv
equities2 = {
'JSE:ADR',
}
eqSym = {
"F",
# "DJI",
# "GDAXI",
# "GSPC",
# "HSI",
# "N225",
# "NYA",
}
register('csv',
viacsv(eqSym)
# 'ch_bundle',
# google_equities(equities2)
)
import pandas as pd
from zipline.data.bundles import register
from zipline.data.bundles.csvdir import csvdir_equities
# zipline bundles
# cp extension.py ~/.zipline/
# zipline ingest -b custom-csv-bundle
start_session = pd.Timestamp('2012-1-3', tz='utc')
end_session = pd.Timestamp('2014-12-31', tz='utc')
register(
'custom-csv-bundle',
csvdir_equities(
['daily'],
'/Users/U201811950/py_study/zipline_demo/csvdir'
),
calendar_name='XNYS', # US equities
start_session=start_session,
end_session=end_session
)
from zipline.data import bundles
from zipline.finance import metrics
from sharadar.loaders.ingest_sharadar import from_quandl
from sharadar.util.metric_daily import default_daily
bundles.register("sharadar", from_quandl(), create_writers=False)
metrics.register('default_daily', default_daily)
import pandas as pd
from zipline.data.bundles import register
from .fut_bundle import futures_bundle
from .settings import start_session, end_session
register(
'futures',
futures_bundle,
calendar_name='NYSE',
start_session=start_session,
end_session=end_session,
)
# register 'tse' calendar
register_calendar('TSE', TehranExchangeCalendar(
start=start_session,
end=end_session))
# register the bundle
"""
command: zipline ingest --bundle tse_stocks
"""
register(
'tse_stocks', # name we select for the bundle
csvdir_equities(
# name of the directory as specified above (named after data frequency)
['daily'],
# path to directory containing the
path
),
calendar_name='TSE',
start_session=start_session,
end_session=end_session
)
"""
command: zipline ingest --bundle 'zipline_bundle_tehran_stocks'
"""
register('zipline_bundle_tehran_stocks',
zipline_bundle_tehran_stocks.ingest, calendar_name='TSE')
"""
https://github.com/quantopian/zipline/issues/2018
"""
import pandas as pd
from zipline.data.bundles import register, india_nse_data
start_session = pd.Timestamp('2005-01-03', tz='utc')
end_session = pd.Timestamp('2020-06-05', tz='utc')
register(
'nse_data',
india_nse_data.nse_data,
calendar_name='XBOM',
start_session=start_session,
end_session=end_session
)
from zipline.utils.factory import create_simulation_parameters
from zipline.data.bundles.core import load
from zipline.data.data_portal import DataPortal
from zipline_cn_databundle.loader import load_market_data
# register SHSZ
from cn_stock_holidays.zipline.default_calendar import shsz_calendar
bundle = 'cn_squant'
start_session_str = '2011-01-05'
register(bundle, squant_bundle, "SHSZ",
pd.Timestamp(start_session_str, tz='utc'),
pd.Timestamp('2016-10-31', tz='utc'))
bundle_data = load(
bundle,
os.environ,
None,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
env = trading.environment = TradingEnvironment(asset_db_path=connstr,
import os
