def init_class_fixtures(cls): super(ExamplesTests, cls).init_class_fixtures() register('test', lambda *args: None) cls.add_class_callback(partial(unregister, 'test')) with tarfile.open(test_resource_path('example_data.tar.gz')) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath( 'example_data/expected_perf/%s' % pd.__version__.replace('.', '-'), ), serialization='pickle', ) # We need to call gc.collect before tearing down our class because we # have a cycle between TradingAlgorithm and AlgorithmSimulator which # ultimately holds a reference to the pipeline engine passed to the # tests here. # This means that we're not guaranteed to have deleted our disk-backed # resource readers (e.g. SQLiteAdjustmentReader) before trying to # delete the tempdir, which causes failures on Windows because Windows # doesn't allow you to delete a file if someone still has an open # handle to that file. # :( cls.add_class_callback(gc.collect)
def make_pipeline_engine(symbols=['SPY', 'TLT'], bundle='etfs_bundle', calendar='NYSE'): register(bundle, symbols) bundle_data = load(bundle) # Set up pipeline engine # Loader for pricing pipeline_loader = USEquityPricingLoader( bundle_data.equity_daily_bar_reader, bundle_data.adjustment_reader, ) def my_dispatcher(column): return loaders[column] def choose_loader(column): if column in USEquityPricing.columns: return pipeline_loader return my_dispatcher(column) trading_calendar = get_calendar(calendar) engine = SimplePipelineEngine( get_loader=choose_loader, calendar=trading_calendar.all_sessions, asset_finder=bundle_data.asset_finder, ) assets = bundle_data.asset_finder.lookup_symbols(symbols, as_of_date=None) return assets, engine
def init_class_fixtures(cls): super(ExamplesTests, cls).init_class_fixtures() register("test", lambda *args: None) cls.add_class_callback(partial(unregister, "test")) with tarfile.open(test_resource_path("example_data.tar.gz")) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath("example_data/expected_perf/%s" % pd.__version__.replace(".", "-")), serialization="pickle", ) # We need to call gc.collect before tearing down our class because we # have a cycle between TradingAlgorithm and AlgorithmSimulator which # ultimately holds a reference to the pipeline engine passed to the # tests here. # This means that we're not guaranteed to have deleted our disk-backed # resource readers (e.g. SQLiteAdjustmentReader) before trying to # delete the tempdir, which causes failures on Windows because Windows # doesn't allow you to delete a file if someone still has an open # handle to that file. # :( cls.add_class_callback(gc.collect)
def main(): register_calendar('BITMEX', AlwaysOpenCalendar()) # The following is necessary because zipline's developer hard-coded NYSE # everywhere in run_algo._run, *DOH*!!! deregister_calendar('NYSE') register_calendar_alias('NYSE', 'BITMEX', force=False) register( 'bitmex', create_bundle( [ Pairs.symbol_btc, #Pairs.symbol_ada, #Pairs.symbol_bch, #Pairs.symbol_eos, #Pairs.symbol_eth, #Pairs.symbol_ltc, #Pairs.symbol_trx, #Pairs.symbol_xrp, ], pd.Timestamp('2017-11-05 00:00:00', tz='utc'), pd.Timestamp('2018-12-18 23:59:59', tz='utc'), ), calendar_name='BITMEX', minutes_per_day=24*60 )
def load_data(): print('Loading data...') from zipline.data.bundles import register from estimize.zipline.data.bundles.yahoo import yahoo_bundle tickers = { 'SPY', } register( 'yahoo', yahoo_bundle(tickers), ) bundles_module.ingest( 'yahoo', os.environ, pd.Timestamp.utcnow(), [], True, ) bundles_module.ingest( 'quantopian-quandl', os.environ, pd.Timestamp.utcnow(), [], True, )
def ingest(bundle, assets_version, show_progress): if bundle == 'tdx': register('tdx', tdx_bundle, 'SHSZ') bundles_module.ingest(bundle, os.environ, pd.Timestamp.utcnow(), assets_version, show_progress, )
def test_bundle(self): zipline_root = self.enter_instance_context(tmp_dir()).path environ = { 'ZIPLINE_ROOT': zipline_root, 'QUANDL_API_KEY': self.api_key, } # custom bundles need to be registered before use or they will not # be recognized register( 'ZacksQuandl', from_zacks_dump( test_resource_path('zacks_samples', 'fictitious.csv'))) ingest('ZacksQuandl', environ=environ) # load bundle now that it has been ingested bundle = load('ZacksQuandl', environ=environ) sids = 0, 1, 2 # check sids match assert_equal(set(bundle.asset_finder.sids), set(sids)) # check asset_{start, end} is the same as {start, end}_date for equity in bundle.asset_finder.retrieve_all(sids): assert_equal(equity.start_date, self.asset_start, msg=equity) assert_equal(equity.end_date, self.asset_end, msg=equity) # get daily OHLCV data from bundle sessions = self.calendar.all_sessions actual = bundle.equity_daily_bar_reader.load_raw_arrays( self.columns, sessions[sessions.get_loc(self.asset_start, 'bfill')], sessions[sessions.get_loc(self.asset_end, 'ffill')], sids, ) # get expected data from csv expected_pricing, expected_adjustments = self._expected_data( bundle.asset_finder, ) # check OHLCV data matches assert_equal(actual, expected_pricing, array_decimal=2) adjustments_for_cols = bundle.adjustment_reader.load_adjustments( self.columns, sessions, pd.Index(sids), ) for column, adjustments, expected in zip(self.columns, adjustments_for_cols, expected_adjustments): assert_equal( adjustments, expected, msg=column, )
def __register_bundle__(self): from zipline.data.bundles import register from zipline.data.bundles import ingest register( self.bundle_name, # name this whatever you like self.tradea_bundle(self.symbol_list), ) ingest(self.bundle_name)
def init_class_fixtures(cls): super(ExamplesTests, cls).init_class_fixtures() register("test", lambda *args: None) cls.add_class_callback(partial(unregister, "test")) with tarfile.open(test_resource_path("example_data.tar.gz")) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath("example_data/expected_perf/%s" % pd.__version__.replace(".", "-")), serialization="pickle", )
def main(job_id, D): print "job_id", job_id, " params:", D equities1 = {} register( 'my-db-bundle', # name this whatever you like viadb(equities1), calendar='SHSZ') parsed = {} parsed['initialize'] = None parsed['handle_data'] = None parsed['before_trading_start'] = None parsed['analyze'] = None parsed['algotext'] = None parsed['defines'] = () parsed['capital_base'] = 1000000 parsed['data'] = None parsed['bundle'] = 'my-db-bundle' #parsed['bundle']='YAHOO' #parsed['bundle_timestamp']=None parsed['bundle_timestamp'] = pd.Timestamp.utcnow() parsed['start'] = Timestamp('2017-03-01 13:30:00+0000', tz='UTC') parsed['end'] = Timestamp('2017-06-01 13:30:00+0000', tz='UTC') parsed['algofile'] = open( '/data/kanghua/workshop/strategy/campaign/hyperparam/example-new/zipline_strategy.py' ) parsed['data_frequency'] = 'daily' parsed['print_algo'] = False parsed['output'] = 'os.devnull' parsed['local_namespace'] = None parsed['environ'] = os.environ parsed['bm_symbol'] = None # Below what we expect spearmint to pass us # parsed['algo_params']=[47,88.7,7.7] # D={} # D['timeperiod']=10 # D['nbdevup']=1.00 # D['nbdevdn']=1.00 parsed['algo_params'] = D perf = _run(**parsed) StartV = perf['portfolio_value'][0] EndV = perf['portfolio_value'][-1] # spearmint wants to minimize so return negative profit OPTIM = (StartV - EndV) return OPTIM
def init_class_fixtures(cls): super(ExamplesTests, cls).init_class_fixtures() register('test', lambda *args: None) cls.add_class_callback(partial(unregister, 'test')) with tarfile.open(test_resource_path('example_data.tar.gz')) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath( 'example_data/expected_perf/%s' % pd.__version__.replace('.', '-'), ), serialization='pickle', )
def main(): register_calendar('CN_FUTURES', CN_FUTURE_CALENDAR(), force=True) deregister_calendar('NYSE') register_calendar_alias('NYSE', 'CN_FUTURES', force=True) register( 'CN_FUTURES', create_bundle( ['IC', 'IH', 'IF', 'RB'], pd.Timestamp('2017-09-20 01:31:00', tz='utc'), pd.Timestamp('2018-10-31 01:31:00', tz='utc'), ), calendar_name='CN_FUTURES', minutes_per_day=4 * 60 + 90 )
def init_class_fixtures(cls): super(ExamplesTests, cls).init_class_fixtures() register('test', lambda *args: None) cls.add_class_callback(partial(unregister, 'test')) with tarfile.open(test_resource_path('example_data.tar.gz')) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath( 'example_data/expected_perf/%s' % pd.__version__.replace('.', '-'), ), serialization='pickle', ) market_data = ('SPY_benchmark.csv', 'treasury_curves.csv') for data in market_data: ensure_file(cls.tmpdir.getpath('example_data/root/data/' + data))
def ingest(bundle, assets, minute, assets_version, show_progress): if bundle == 'tdx': if assets: if not os.path.exists(assets): raise FileNotFoundError df = pd.read_csv(assets, names=['symbol', 'name'], dtype=str, encoding='utf8') register('tdx', partial(tdx_bundle, df, minute), 'SHSZ') else: register('tdx', partial(tdx_bundle, None, minute), 'SHSZ') bundles_module.ingest( bundle, os.environ, pd.Timestamp.utcnow(), assets_version, show_progress, )
def register_cn_bundle_from_yahoo(name, cache=True): """ register a new bundle of stocks from chinese market from yahoo :param name: the name of bundle :return: register result """ symbol_list = get_filtered_symbols(cache) return register( name, yahoo_equities(dict(list(zip(symbol_list, symbol_list)))), )
def init_class_fixtures(cls): super(ExamplesTests, cls).init_class_fixtures() register("test", lambda *args: None) cls.add_class_callback(partial(unregister, "test")) with tarfile.open(test_resource_path("example_data.tar.gz")) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath( "example_data/expected_perf/%s" % pd.__version__.replace(".", "-"), ), serialization="pickle", ) cls.no_benchmark_expected_perf = { example_name: cls._no_benchmark_expectations_applied(expected_perf.copy()) for example_name, expected_perf in cls.expected_perf.items() }
def init_class_fixtures(cls): super().init_class_fixtures() register('test', lambda *args: None) cls.add_class_callback(partial(unregister, 'test')) with tarfile.open(test_resource_path('example_data.tar.gz')) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath( 'example_data/expected_perf/%s' % pd.__version__.replace('.', '-'), ), serialization='pickle', ) cls.no_benchmark_expected_perf = { example_name: cls._no_benchmark_expectations_applied(expected_perf.copy()) for example_name, expected_perf in cls.expected_perf.items() }
def target_ingest(assets, ingest_minute=False): import cn_stock_holidays.zipline.default_calendar if assets: if not os.path.exists(assets): raise FileNotFoundError df = pd.read_csv(assets, names=['symbol', 'name'], dtype=str, encoding='utf8') register('tdx', partial(tdx_bundle, df[:1], ingest_minute), 'SHSZ') else: df = pd.DataFrame({'symbol': ['000001'], 'name': ['平安银行']}) register('tdx', partial(tdx_bundle, df, ingest_minute), 'SHSZ') bundles_module.ingest( 'tdx', os.environ, pd.Timestamp.utcnow(), show_progress=True, )
def register_tdx(assets=None, minute=False, start=None, overwrite=False, end=None): calendar = get_calendar('SHSZ') if not end: now = pd.to_datetime('now') if now.time() < pd.to_datetime('15:30').tz_localize( 'Asia/Shanghai').tz_convert("UTC").time(): end = now.date() - pd.Timedelta('24 H') end = calendar.all_sessions[calendar.all_sessions.get_loc( end, method='ffill')] else: end = now.date() if start: if not calendar.is_session(start): start = calendar.all_sessions[searchsorted(calendar.all_sessions, start)] register('tdx', partial(tdx_bundle, assets, minute, overwrite), 'SHSZ', start, end)
class YahooConfig(Config): YAHOO_TICKERS = { 'SPY', } register( 'yahoo', yahoo_bundle(YAHOO_TICKERS), ) def __init__(self): super(YahooConfig, self).__init__(bundle_name='yahoo')
def _setup_class(request, tmpdir_factory): request.cls.tmp_path = tmpdir_factory.mktemp("tmp") request.cls.tmpdir = str(request.cls.tmp_path) register("test", lambda *args: None) with tarfile.open(join(TEST_RESOURCE_PATH, "example_data.tar.gz")) as tar: tar.extractall(request.cls.tmpdir) request.cls.expected_perf_dirs = listdir( join( str(request.cls.tmp_path), "example_data", "expected_perf", )) if PANDAS_VERSION not in request.cls.expected_perf_dirs: warnings.warn( "No data stored matches the current version of pandas. " "Consider including the data using rebuild_example_data", ) yield partial(unregister, "test")
def init_class_fixtures(cls): super(ExamplesTests, cls).init_class_fixtures() register('test', lambda *args: None) cls.add_class_callback(partial(unregister, 'test')) with tarfile.open(test_resource_path('example_data.tar.gz')) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath( 'example_data/expected_perf/%s' % pd.__version__.replace('.', '-'), ), serialization='pickle', ) market_data = ('SPY_benchmark.csv', 'treasury_curves.csv') for data in market_data: update_modified_time( cls.tmpdir.getpath( 'example_data/root/data/' + data ) )
# start_date += timedelta(days=1) start_date = end_date - timedelta(days=365) while not cal.is_session(start_date): start_date -= timedelta(days=1) initialize_client() import time start_time = time.time() register( 'alpaca_api', # api_to_bundle(interval=['1d', '1m']), # api_to_bundle(interval=['1m']), api_to_bundle(interval=['1d']), calendar_name='NYSE', start_session=start_date, end_session=end_date) assets_version = ((), )[0] # just a weird way to create an empty tuple bundles_module.ingest( "alpaca_api", os.environ, assets_versions=assets_version, show_progress=True, ) print(f"--- It took {timedelta(seconds=time.time() - start_time)} ---")
@property def name(self): return "bitxex" @property def tz(self): return timezone('UTC') @property def open_time(self): return time(0, 0) @property def close_time(self): return time(23, 59) @lazyval def day(self): return CustomBusinessDay(weekmask='Mon Tue Wed Thu Fri Sat Sun') if __name__ == '__main__': register_calendar('bitxex', BitmexCalendar()) register('bitxex', bitmex(['XBTUSD']), calendar_name='bitxex', start_session=pd.Timestamp('2018-05-20', tz='utc'), end_session=pd.Timestamp('2018-06-10', tz='utc'), minutes_per_day=24 * 60)
'Predict Factor': predict.downsample('week_start'), 'longs': longs.downsample('week_start'), 'shorts': shorts.downsample('week_start'), 'predict_rank': predict_rank.downsample('week_start'), } pipe = Pipeline( columns=pipe_final_columns, screen=long_short_screen, ) return pipe ############################################# bundle ############################################# equities1 = {} register( 'my-db-bundle', # name this whatever you like viadb(equities1), calendar='SHSZ') bundle = 'my-db-bundle' bundle_timestamp = pd.Timestamp.utcnow() environ = os.environ bundle_data = load( bundle, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) #print prefix, connstr
from zipline.data.bundles import register from zipline.data.bundles.google import google_equities, viacsv equities2 = { 'JSE:ADR', } eqSym = { "F", # "DJI", # "GDAXI", # "GSPC", # "HSI", # "N225", # "NYA", } register('csv', viacsv(eqSym) # 'ch_bundle', # google_equities(equities2) )
import pandas as pd from zipline.data.bundles import register from zipline.data.bundles.csvdir import csvdir_equities # zipline bundles # cp extension.py ~/.zipline/ # zipline ingest -b custom-csv-bundle start_session = pd.Timestamp('2012-1-3', tz='utc') end_session = pd.Timestamp('2014-12-31', tz='utc') register( 'custom-csv-bundle', csvdir_equities( ['daily'], '/Users/U201811950/py_study/zipline_demo/csvdir' ), calendar_name='XNYS', # US equities start_session=start_session, end_session=end_session )
from zipline.data import bundles from zipline.finance import metrics from sharadar.loaders.ingest_sharadar import from_quandl from sharadar.util.metric_daily import default_daily bundles.register("sharadar", from_quandl(), create_writers=False) metrics.register('default_daily', default_daily)
import pandas as pd from zipline.data.bundles import register from .fut_bundle import futures_bundle from .settings import start_session, end_session register( 'futures', futures_bundle, calendar_name='NYSE', start_session=start_session, end_session=end_session, )
# register 'tse' calendar register_calendar('TSE', TehranExchangeCalendar( start=start_session, end=end_session)) # register the bundle """ command: zipline ingest --bundle tse_stocks """ register( 'tse_stocks', # name we select for the bundle csvdir_equities( # name of the directory as specified above (named after data frequency) ['daily'], # path to directory containing the path ), calendar_name='TSE', start_session=start_session, end_session=end_session ) """ command: zipline ingest --bundle 'zipline_bundle_tehran_stocks' """ register('zipline_bundle_tehran_stocks', zipline_bundle_tehran_stocks.ingest, calendar_name='TSE') """ https://github.com/quantopian/zipline/issues/2018 """
import pandas as pd from zipline.data.bundles import register, india_nse_data start_session = pd.Timestamp('2005-01-03', tz='utc') end_session = pd.Timestamp('2020-06-05', tz='utc') register( 'nse_data', india_nse_data.nse_data, calendar_name='XBOM', start_session=start_session, end_session=end_session )
from zipline.utils.factory import create_simulation_parameters from zipline.data.bundles.core import load from zipline.data.data_portal import DataPortal from zipline_cn_databundle.loader import load_market_data # register SHSZ from cn_stock_holidays.zipline.default_calendar import shsz_calendar bundle = 'cn_squant' start_session_str = '2011-01-05' register(bundle, squant_bundle, "SHSZ", pd.Timestamp(start_session_str, tz='utc'), pd.Timestamp('2016-10-31', tz='utc')) bundle_data = load( bundle, os.environ, None, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) env = trading.environment = TradingEnvironment(asset_db_path=connstr,
import os