def get_tickers_from_bundle(bundle_name):
"""Gets a list of tickers from a given bundle"""
bundle_data = load(bundle_name, os.environ, None)
# get a list of all sids
lifetimes = bundle_data.asset_finder._compute_asset_lifetimes()
all_sids = lifetimes.sid
# retreive all assets in the bundle
all_assets = bundle_data.asset_finder.retrieve_all(all_sids)
# return only tickers
return map(lambda x: (x.symbol, x.sid), all_assets)
def make_pipeline_engine(bundle, data_dates):
"""Creates a pipeline engine for the dates in data_dates.
Using this allows usage very similar to run_pipeline in Quantopian's env."""
bundle_data = load(bundle, os.environ, None)
pipeline_loader = USEquityPricingLoader(bundle_data.equity_daily_bar_reader, bundle_data.adjustment_reader)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError("No PipelineLoader registered for column %s." % column)
# set up pipeline
cal = bundle_data.equity_daily_bar_reader.trading_calendar.all_sessions
cal2 = cal[(cal >= data_dates[0]) & (cal <= data_dates[1])]
spe = SimplePipelineEngine(get_loader=choose_loader,
calendar=cal2,
asset_finder=bundle_data.asset_finder)
return spe
def get_zipline_dp(bundle=None, calendar=None):
'''
Returns `zipline` data portal, used for symbol lookups and obtaining data.
:rtype: `zipline.data.data_portal.DataPortal`
:param bundle: optionally specify the `zipline` data bundle to use
:param calendar: optionally specify the `zipline` calendar to use
:type bundle: `zipline.data.bundles.core.BundleData`
:type calendar: `zipline.utils.calendars.exchange_calendar_nyse` type
'''
z = _ZIPLINE_QUANDL_BUNDLE
bundle = load(z['name']) if bundle is None else bundle
calendar = get_calendar(z['cal']) if calendar is None else calendar
return DataPortal(
bundle.asset_finder,
calendar,
first_trading_day=bundle.equity_minute_bar_reader.first_trading_day,
equity_minute_reader=bundle.equity_minute_bar_reader,
equity_daily_reader=bundle.equity_daily_bar_reader,
adjustment_reader=bundle.adjustment_reader,
)
def get_zipline_dp(bundle=None, calendar=None):
'''
Returns `zipline` data portal, used for symbol lookups and obtaining data.
:rtype: `zipline.data.data_portal.DataPortal`
:param bundle: optionally specify the `zipline` data bundle to use
:param calendar: optionally specify the `zipline` calendar to use
:type bundle: `zipline.data.bundles.core.BundleData`
:type calendar: `zipline.utils.calendars.exchange_calendar_nyse` type
'''
z = _ZIPLINE_QUANDL_BUNDLE
bundle = load(z['name']) if bundle is None else bundle
calendar = get_calendar(z['cal']) if calendar is None else calendar
return DataPortal(
bundle.asset_finder,
calendar,
first_trading_day=bundle.equity_minute_bar_reader.first_trading_day,
equity_minute_reader=bundle.equity_minute_bar_reader,
equity_daily_reader=bundle.equity_daily_bar_reader,
adjustment_reader=bundle.adjustment_reader,
)
)
return pipe
############################################# bundle #############################################
equities1 = {}
register(
'my-db-bundle', # name this whatever you like
viadb(equities1),
calendar='SHSZ')
bundle = 'my-db-bundle'
bundle_timestamp = pd.Timestamp.utcnow()
environ = os.environ
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
#print prefix, connstr
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url), )
############################################# trading_environment #############################################
trading_calendar = get_calendar("SHSZ")
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
algotext, defines, data_frequency, capital_base, data, bundle,
bundle_timestamp, start, end, output, print_algo, local_namespace,
environ):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign, )
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' %
(name, e), )
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if bundle is not None:
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url), )
env = TradingEnvironment(asset_db_path=connstr)
data = DataPortal(
env,
equity_minute_reader=bundle_data.minute_bar_reader,
equity_daily_reader=bundle_data.daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
perf = TradingAlgorithm(namespace=namespace,
capital_base=capital_base,
start=start,
end=end,
env=env,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': algofile.name,
'script': algotext,
}).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
from zipline.utils.factory import create_simulation_parameters
from zipline.gens.brokers.broker import Broker
from zipline.data.data_portal_live import DataPortalLive
from zipline.data.cn_loader import load_market_data
from zipline.algorithm_live import LiveTradingAlgorithm
from zipline.algorithm import TradingAlgorithm
import os
import re
import sys
import warnings
from runpy import run_path
environ = os.environ
bundle_data = load('tdx')
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
first_trading_day = \
bundle_data.equity_minute_bar_reader.first_trading_day
env = TradingEnvironment(load=load_market_data,
bm_symbol='000300',
asset_db_path=connstr,
environ=environ)
calendar = get_calendar('SHSZ')
data = DataPortal(
env.asset_finder,
calendar,
from zipline.utils.factory import create_simulation_parameters
from zipline.gens.brokers.broker import Broker
from zipline.data.data_portal_live import DataPortalLive
from zipline.data.cn_loader import load_market_data
from zipline.algorithm_live import LiveTradingAlgorithm
from zipline.algorithm import TradingAlgorithm
import os
import re
import sys
import warnings
from runpy import run_path
environ = os.environ
bundle_data = load('tdx')
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
first_trading_day = \
bundle_data.equity_minute_bar_reader.first_trading_day
env = TradingEnvironment(load=load_market_data, bm_symbol='000300', asset_db_path=connstr, environ=environ)
calendar = get_calendar('SHSZ')
data = DataPortal(
env.asset_finder, calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
# register SHSZ
from cn_stock_holidays.zipline.default_calendar import shsz_calendar
bundle = 'cn_squant'
start_session_str = '2011-01-05'
register(bundle, squant_bundle, "SHSZ",
pd.Timestamp(start_session_str, tz='utc'),
pd.Timestamp('2016-10-31', tz='utc'))
bundle_data = load(
bundle,
os.environ,
None,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
env = trading.environment = TradingEnvironment(asset_db_path=connstr,
trading_calendar=shsz_calendar,
bm_symbol='000001.SS',
load=load_market_data)
def _run(handle_data,
initialize,
before_trading_start,
analyze,
algofile,
algotext,
defines,
data_frequency,
capital_base,
data,
bundle,
bundle_timestamp,
start,
end,
output,
print_algo,
local_namespace,
environ):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if bundle is not None:
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url),
)
env = TradingEnvironment(asset_db_path=connstr)
first_trading_day =\
bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
env.asset_finder, get_calendar("NYSE"),
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError(
"No PipelineLoader registered for column %s." % column
)
perf = TradingAlgorithm(
namespace=namespace,
capital_base=capital_base,
start=start,
end=end,
env=env,
get_pipeline_loader=choose_loader,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}
).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
def bundle_data(self):
return load(self.bundle_name)
def make_choose_loader(pl_loader):
def cl(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError("No PipelineLoader registered for column %s." %
column)
return cl
if __name__ == '__main__':
# load the bundle
bundle_data = load('quantopian-quandl', os.environ, None)
cal = bundle_data.equity_daily_bar_reader.trading_calendar.all_sessions
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader, bundle_data.adjustment_reader)
choose_loader = make_choose_loader(pipeline_loader)
env = TradingEnvironment(asset_db_path=parse_sqlite_connstr(
bundle_data.asset_finder.engine.url))
data = DataPortal(
env.asset_finder,
get_calendar("NYSE"),
first_trading_day=bundle_data.equity_minute_bar_reader.
first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
def _run(handle_data,
initialize,
before_trading_start,
analyze,
algofile,
algotext,
defines,
data_frequency,
capital_base,
data,
bundle,
bundle_timestamp,
start,
end,
output,
trading_calendar,
print_algo,
local_namespace,
environ):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if trading_calendar is None:
trading_calendar = get_calendar('NYSE')
if bundle is not None: #ronz bundle='quantopian-quandl'
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url),
)
env = TradingEnvironment(asset_db_path=connstr, environ=environ) #ronz asset_db_path='/home/gqian/.zipline/data/quantopian-quandl/2017-09-22T11;56;54.022199/assets-6.sqlite'
first_trading_day =\
bundle_data.equity_minute_bar_reader.first_trading_day #ronz Timestamp('1990-01-02 00:00:00+0000', tz='UTC')
data = DataPortal( #ronz create DataPortal as the data container used all over the application data_portal.py
env.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
pipeline_loader = USEquityPricingLoader( #ronz pipeline loader engine for continuous future ?? cannot find much usage
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError(
"No PipelineLoader registered for column %s." % column
)
else:
env = TradingEnvironment(environ=environ)
choose_loader = None
perf = TradingAlgorithm(
namespace=namespace, #ronz input algofile has no namespace
env=env, #ronz TradingEnvironment that includes, trading_calendar/load bm data/load input database engine/asset finder/asset writer
get_pipeline_loader=choose_loader,
trading_calendar=trading_calendar,
sim_params=create_simulation_parameters( #ronz returns SimulationParameters obj that contains all simulation param
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
trading_calendar=trading_calendar,
),
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {#ronz go this branch, all these "api_methods" are got direct from algofilei->algotext now, hence above are passed down from cmdline->main->_run->here as NONE
'algo_filename': getattr(algofile, 'name', '<algorithm>'), #ronz 'buyapple.py' '<algorithm>' is default
'script': algotext,
}
).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
import pandas as pd
from cswd.utils import sanitize_dates
from zipline.pipeline import Pipeline
from zipline.pipeline.engine import SimplePipelineEngine
from zipline.pipeline.data import USEquityPricing
from zipline.pipeline.loaders import USEquityPricingLoader
from zipline.pipeline import Fundamentals
from zipline.data.bundles.core import load
from zipline.pipeline.loaders.blaze import global_loader
bundle = 'cnstock'
bundle_data = load(bundle)
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
elif Fundamentals.has_column(column):
return global_loader
raise ValueError("No PipelineLoader registered for column %s." % column)
import os
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
algotext, defines, data_frequency, capital_base, data, bundle,
bundle_timestamp, start, end, output, trading_calendar, print_algo,
local_namespace, environ, broker, state_filename,
realtime_bar_target):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign, )
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' %
(name, e), )
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if not trading_calendar:
trading_calendar = get_calendar('SHSZ')
elif isinstance(trading_calendar, str):
trading_calendar = get_calendar(trading_calendar)
if bundle is not None:
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url), )
env = TradingEnvironment(load=load_market_data,
bm_symbol='000300',
asset_db_path=connstr,
environ=environ)
first_trading_day = \
bundle_data.equity_minute_bar_reader.first_trading_day
DataPortalClass = (partial(DataPortalLive, broker)
if broker else DataPortal)
data = DataPortalClass(
env.asset_finder,
trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError("No PipelineLoader registered for column %s." %
column)
else:
env = TradingEnvironment(
environ=environ,
load=load_market_data,
bm_symbol='000300',
)
choose_loader = None
emission_rate = 'daily' # TODO why daily default
if broker:
emission_rate = 'minute'
start = pd.Timestamp.utcnow()
end = start + pd.Timedelta('2 day')
TradingAlgorithmClass = (partial(LiveTradingAlgorithm,
broker=broker,
state_filename=state_filename,
realtime_bar_target=realtime_bar_target)
if broker else TradingAlgorithm)
perf = TradingAlgorithmClass(
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
trading_calendar=trading_calendar,
sim_params=create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
emission_rate=emission_rate,
data_frequency=data_frequency,
trading_calendar=trading_calendar,
),
fundamental_reader=bundle_data.fundamental_reader,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
def get_trading_days(bundle, start_date, end_date):
"""Gets the trading days between start_date and end_date inclusive, for a given bundle.
Will also return the trading calendar."""
bundle_data = load(bundle, os.environ, None)
cal = bundle_data.equity_daily_bar_reader.trading_calendar.all_sessions
return cal[(cal >= start_date) & (cal <= end_date)], cal
