def create_data_portal(env, tempdir, sim_params, sids, adjustment_reader=None): if sim_params.data_frequency == "daily": daily_path = write_daily_data(tempdir, sim_params, sids) equity_daily_reader = BcolzDailyBarReader(daily_path) return DataPortal( env, equity_daily_reader=equity_daily_reader, adjustment_reader=adjustment_reader ) else: minutes = env.minutes_for_days_in_range( sim_params.first_open, sim_params.last_close ) minute_path = write_minute_data(env, tempdir, minutes, sids) equity_minute_reader = BcolzMinuteBarReader(minute_path) return DataPortal( env, equity_minute_reader=equity_minute_reader, adjustment_reader=adjustment_reader )
def get_prices(numdays=1000): now = Timestamp.utcnow() bundle = load('quantopian-quandl', os.environ, now) all_assets = bundle.asset_finder.retrieve_all(bundle.asset_finder.sids) symbols = set( str(asset.symbol) for asset in bundle.asset_finder.retrieve_all( bundle.asset_finder.equities_sids)) quandl_data = DataPortal( asset_finder=bundle.asset_finder, trading_calendar=get_calendar('NYSE'), first_trading_day=bundle.equity_daily_bar_reader.first_trading_day, equity_minute_reader=bundle.equity_minute_bar_reader, equity_daily_reader=bundle.equity_daily_bar_reader, adjustment_reader=bundle.adjustment_reader) end_dt = pd.Timestamp('2018-03-31', tz='utc') quandl_symbols = [] for ticker in symbols: try: quandl_symbols.append( quandl_data.asset_finder.lookup_symbol(ticker, end_dt)) except: continue quandl_pricing = quandl_data.get_history_window(quandl_symbols, end_dt, numdays, '1d', 'close', 'daily') quandl_pricing.columns = [c.symbol for c in quandl_pricing.columns] return quandl_pricing
def prices(assets, start, end, field='close', offset=0): """ Get price data for assets between start and end. """ start = trading_date(start) end = trading_date(end) bundle = load_sharadar_bundle() trading_calendar = bundle.equity_daily_bar_reader.trading_calendar if offset > 0: start = trading_calendar.sessions_window(start, -offset)[0] bar_count = trading_calendar.session_distance(start, end) data_portal = DataPortal( bundle.asset_finder, trading_calendar=trading_calendar, first_trading_day=start, equity_daily_reader=bundle.equity_daily_bar_reader, adjustment_reader=bundle.adjustment_reader) df = data_portal.get_history_window(assets=assets, end_dt=end, bar_count=bar_count, frequency='1d', field=field, data_frequency='daily') return df if len(assets) > 1 else df.squeeze()
def create_data_portal(asset_finder, tempdir, sim_params, sids, trading_calendar, adjustment_reader=None): if sim_params.data_frequency == "daily": daily_path = write_daily_data(tempdir, sim_params, sids, trading_calendar) equity_daily_reader = BcolzDailyBarReader(daily_path) return DataPortal( asset_finder, trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, equity_daily_reader=equity_daily_reader, adjustment_reader=adjustment_reader ) else: minutes = trading_calendar.minutes_in_range( sim_params.first_open, sim_params.last_close ) minute_path = write_minute_data(trading_calendar, tempdir, minutes, sids) equity_minute_reader = BcolzMinuteBarReader(minute_path) return DataPortal( asset_finder, trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader, adjustment_reader=adjustment_reader )
class TestDataPortal(WithTradingEnvironment, ZiplineTestCase): def init_instance_fixtures(self): super(TestDataPortal, self).init_instance_fixtures() self.data_portal = DataPortal(self.env.asset_finder, self.trading_schedule, first_trading_day=None) def test_bar_count_for_simple_transforms(self): # July 2015 # Su Mo Tu We Th Fr Sa # 1 2 3 4 # 5 6 7 8 9 10 11 # 12 13 14 15 16 17 18 # 19 20 21 22 23 24 25 # 26 27 28 29 30 31 # half an hour into july 9, getting a 4-"day" window should get us # all the minutes of 7/6, 7/7, 7/8, and 31 minutes of 7/9 july_9_dt = self.trading_schedule.start_and_end( pd.Timestamp("2015-07-09", tz='UTC') )[0] + Timedelta("30 minutes") self.assertEqual( (3 * 390) + 31, self.data_portal._get_minute_count_for_transform(july_9_dt, 4) ) # November 2015 # Su Mo Tu We Th Fr Sa # 1 2 3 4 5 6 7 # 8 9 10 11 12 13 14 # 15 16 17 18 19 20 21 # 22 23 24 25 26 27 28 # 29 30 # nov 26th closed # nov 27th was an early close # half an hour into nov 30, getting a 4-"day" window should get us # all the minutes of 11/24, 11/25, 11/27 (half day!), and 31 minutes # of 11/30 nov_30_dt = self.trading_schedule.start_and_end( pd.Timestamp("2015-11-30", tz='UTC') )[0] + Timedelta("30 minutes") self.assertEqual( 390 + 390 + 210 + 31, self.data_portal._get_minute_count_for_transform(nov_30_dt, 4) )
def test_asset_IPOed_same_day(self): # gotta get some minute data up in here. # add sid 4 for a couple of days minutes = self.trading_calendar.minutes_for_sessions_in_range( self.sim_params.sessions[0], self.sim_params.sessions[5] ) tmp_reader = tmp_bcolz_equity_minute_bar_reader( self.trading_calendar, self.trading_calendar.all_sessions, create_minute_bar_data(minutes, [2]), ) with tmp_reader as reader: data_portal = DataPortal( self.env.asset_finder, self.trading_calendar, first_trading_day=reader.first_trading_day, equity_minute_reader=reader, equity_daily_reader=self.bcolz_equity_daily_bar_reader, adjustment_reader=self.adjustment_reader, ) source = BenchmarkSource( 2, self.env, self.trading_calendar, self.sim_params.sessions, data_portal ) days_to_use = self.sim_params.sessions # first value should be 0.0, coming from daily data self.assertAlmostEquals(0.0, source.get_value(days_to_use[0])) manually_calculated = data_portal.get_history_window( [2], days_to_use[-1], len(days_to_use), "1d", "close", "daily", )[2].pct_change() for idx, day in enumerate(days_to_use[1:]): self.assertEqual( source.get_value(day), manually_calculated[idx + 1] )
def test_orders_stop(self, name, order_data, event_data, expected): data = order_data data['asset'] = self.ASSET133 order = Order(**data) if expected['transaction']: expected['transaction']['asset'] = self.ASSET133 event_data['asset'] = self.ASSET133 assets = ( (133, pd.DataFrame( { 'open': [event_data['open']], 'high': [event_data['high']], 'low': [event_data['low']], 'close': [event_data['close']], 'volume': [event_data['volume']], }, index=[pd.Timestamp('2006-01-05 14:31', tz='UTC')], )), ) days = pd.date_range( start=normalize_date(self.minutes[0]), end=normalize_date(self.minutes[-1]) ) with tmp_bcolz_equity_minute_bar_reader( self.trading_calendar, days, assets) as reader: data_portal = DataPortal( self.env.asset_finder, self.trading_calendar, first_trading_day=reader.first_trading_day, equity_minute_reader=reader, ) slippage_model = VolumeShareSlippage() try: dt = pd.Timestamp('2006-01-05 14:31', tz='UTC') bar_data = BarData( data_portal, lambda: dt, self.sim_params.data_frequency, self.trading_calendar, NoRestrictions(), ) _, txn = next(slippage_model.simulate( bar_data, self.ASSET133, [order], )) except StopIteration: txn = None if expected['transaction'] is None: self.assertIsNone(txn) else: self.assertIsNotNone(txn) for key, value in expected['transaction'].items(): self.assertEquals(value, txn[key])
def test_orders_stop(self, name, order_data, event_data, expected): tempdir = TempDirectory() try: data = order_data data['sid'] = self.ASSET133 order = Order(**data) assets = { 133: pd.DataFrame({ "open": [event_data["open"]], "high": [event_data["high"]], "low": [event_data["low"]], "close": [event_data["close"]], "volume": [event_data["volume"]], "dt": [pd.Timestamp('2006-01-05 14:31', tz='UTC')] }).set_index("dt") } write_bcolz_minute_data( self.env, pd.date_range( start=normalize_date(self.minutes[0]), end=normalize_date(self.minutes[-1]) ), tempdir.path, assets ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( self.env, equity_minute_reader=equity_minute_reader, ) slippage_model = VolumeShareSlippage() try: dt = pd.Timestamp('2006-01-05 14:31', tz='UTC') bar_data = BarData(data_portal, lambda: dt, 'minute') _, txn = next(slippage_model.simulate( bar_data, self.ASSET133, [order], )) except StopIteration: txn = None if expected['transaction'] is None: self.assertIsNone(txn) else: self.assertIsNotNone(txn) for key, value in expected['transaction'].items(): self.assertEquals(value, txn[key]) finally: tempdir.cleanup()
def set_bundle(name, calendar='NYSE'): global trading_calendar global bundle global bundle_data global engine global choose_loader global data bundle = name trading_calendar = get_calendar(calendar) bundle_data = bundles.load(bundle) engine = SimplePipelineEngine( get_loader=choose_loader, calendar=trading_calendar.all_sessions, asset_finder=bundle_data.asset_finder, ) data = DataPortal( bundle_data.asset_finder, trading_calendar=trading_calendar, first_trading_day=bundle_data.equity_daily_bar_reader.first_trading_day, equity_minute_reader=None, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, )
def test_get_value_during_non_market_hours(self): # make sure that if we try to get the OHLCV values of ASSET1 during # non-market hours, we don't get the previous market minute's values futures_cal = get_calendar("us_futures") data_portal = DataPortal( self.env.asset_finder, futures_cal, first_trading_day=self.DATA_PORTAL_FIRST_TRADING_DAY, equity_minute_reader=self.bcolz_equity_minute_bar_reader, ) bar_data = BarData( data_portal, lambda: pd.Timestamp("2016-01-06 3:15", tz="US/Eastern"), "minute", futures_cal ) self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "open"))) self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "high"))) self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "low"))) self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "close"))) self.assertEqual(0, bar_data.current(self.ASSET1, "volume")) # price should still forward fill self.assertEqual(390, bar_data.current(self.ASSET1, "price"))
def _data_portal(bundle=DEFAULT_BUNDLE): """Create a DataPortal for the given bundle. Parameters ---------- bundle : str The name of the bundle to create a pipeline engine for. Returns ------- engine : zipline.pipleine.engine.SimplePipelineEngine The pipeline engine which can run pipelines against the bundle. calendar : zipline.utils.calendars.TradingCalendar The trading calendar for the bundle. """ bundle_data = bundles.load(bundle) calendar = bundle_data.equity_daily_bar_reader.trading_calendar finder = bundle_data.asset_finder data_portal = DataPortal( finder, calendar, calendar.first_session, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader) return data_portal, calendar
class TestDataPortal(WithTradingEnvironment, ZiplineTestCase): def init_instance_fixtures(self): super(TestDataPortal, self).init_instance_fixtures() self.data_portal = DataPortal(self.env.asset_finder, self.trading_schedule, first_trading_day=None) def test_bar_count_for_simple_transforms(self): # July 2015 # Su Mo Tu We Th Fr Sa # 1 2 3 4 # 5 6 7 8 9 10 11 # 12 13 14 15 16 17 18 # 19 20 21 22 23 24 25 # 26 27 28 29 30 31 # half an hour into july 9, getting a 4-"day" window should get us # all the minutes of 7/6, 7/7, 7/8, and 31 minutes of 7/9 july_9_dt = self.trading_schedule.start_and_end( pd.Timestamp("2015-07-09", tz='UTC'))[0] + Timedelta("30 minutes") self.assertEqual( (3 * 390) + 31, self.data_portal._get_minute_count_for_transform(july_9_dt, 4)) # November 2015 # Su Mo Tu We Th Fr Sa # 1 2 3 4 5 6 7 # 8 9 10 11 12 13 14 # 15 16 17 18 19 20 21 # 22 23 24 25 26 27 28 # 29 30 # nov 26th closed # nov 27th was an early close # half an hour into nov 30, getting a 4-"day" window should get us # all the minutes of 11/24, 11/25, 11/27 (half day!), and 31 minutes # of 11/30 nov_30_dt = self.trading_schedule.start_and_end( pd.Timestamp("2015-11-30", tz='UTC'))[0] + Timedelta("30 minutes") self.assertEqual( 390 + 390 + 210 + 31, self.data_portal._get_minute_count_for_transform(nov_30_dt, 4))
def test_asset_IPOed_same_day(self): # gotta get some minute data up in here. # add sid 4 for a couple of days minutes = self.trading_calendar.minutes_for_sessions_in_range( self.sim_params.sessions[0], self.sim_params.sessions[5] ) tmp_reader = tmp_bcolz_equity_minute_bar_reader( self.trading_calendar, self.trading_calendar.all_sessions, create_minute_bar_data(minutes, [2]), ) with tmp_reader as reader: data_portal = DataPortal( self.env.asset_finder, self.trading_calendar, first_trading_day=reader.first_trading_day, equity_minute_reader=reader, equity_daily_reader=self.bcolz_equity_daily_bar_reader, adjustment_reader=self.adjustment_reader, ) source = BenchmarkSource( 2, self.env, self.trading_calendar, self.sim_params.sessions, data_portal ) days_to_use = self.sim_params.sessions # first value should be 0.0, coming from daily data self.assertAlmostEquals(0.0, source.get_value(days_to_use[0])) manually_calculated = data_portal.get_history_window( [2], days_to_use[-1], len(days_to_use), "1d", "close", )[2].pct_change() for idx, day in enumerate(days_to_use[1:]): self.assertEqual( source.get_value(day), manually_calculated[idx + 1] )
def setUpClass(cls): setup_logger(cls) cls.env = trading.TradingEnvironment() cls.sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-05", tz='UTC'), end=pd.Timestamp("2006-01-06", tz='UTC')) cls.env.write_data( equities_data={ 24: { 'start_date': cls.sim_params.trading_days[0], 'end_date': cls.env.next_trading_day(cls.sim_params.trading_days[-1]) }, 25: { 'start_date': cls.sim_params.trading_days[0], 'end_date': cls.env.next_trading_day(cls.sim_params.trading_days[-1]) } }) cls.tempdir = TempDirectory() assets = { 24: pd.DataFrame({ "open": [50, 50], "high": [50, 50], "low": [50, 50], "close": [50, 50], "volume": [100, 400], "day": [day.value for day in cls.sim_params.trading_days] }), 25: pd.DataFrame({ "open": [50, 50], "high": [50, 50], "low": [50, 50], "close": [50, 50], "volume": [100, 400], "day": [day.value for day in cls.sim_params.trading_days] }) } path = os.path.join(cls.tempdir.path, "tempdata.bcolz") DailyBarWriterFromDataFrames(assets).write(path, cls.sim_params.trading_days, assets) equity_daily_reader = BcolzDailyBarReader(path) cls.data_portal = DataPortal( cls.env, equity_daily_reader=equity_daily_reader, )
def get_data_portal(cls, bundle_data, trading_calendar): return DataPortal( bundle_data.asset_finder, trading_calendar=trading_calendar, first_trading_day=bundle_data.equity_daily_bar_reader. first_trading_day, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader)
def setUpClass(cls): cls.tempdir = TempDirectory() cls.env = TradingEnvironment() cls.sim_params = SimulationParameters( period_start=pd.Timestamp("2006-01-05 14:31", tz="utc"), period_end=pd.Timestamp("2006-01-05 14:36", tz="utc"), capital_base=1.0e5, data_frequency="minute", emission_rate='daily', env=cls.env ) cls.sids = [133] cls.minutes = pd.DatetimeIndex( start=pd.Timestamp("2006-01-05 14:31", tz="utc"), end=pd.Timestamp("2006-01-05 14:35", tz="utc"), freq="1min" ) assets = { 133: pd.DataFrame({ "open": np.array([3.0, 3.0, 3.5, 4.0, 3.5]), "high": np.array([3.15, 3.15, 3.15, 3.15, 3.15]), "low": np.array([2.85, 2.85, 2.85, 2.85, 2.85]), "close": np.array([3.0, 3.5, 4.0, 3.5, 3.0]), "volume": [2000, 2000, 2000, 2000, 2000], "dt": cls.minutes }).set_index("dt") } write_bcolz_minute_data( cls.env, pd.date_range( start=normalize_date(cls.minutes[0]), end=normalize_date(cls.minutes[-1]) ), cls.tempdir.path, assets ) cls.env.write_data(equities_data={ 133: { "start_date": pd.Timestamp("2006-01-05", tz='utc'), "end_date": pd.Timestamp("2006-01-07", tz='utc') } }) cls.ASSET133 = cls.env.asset_finder.retrieve_asset(133) cls.data_portal = DataPortal( cls.env, equity_minute_reader=BcolzMinuteBarReader(cls.tempdir.path), )
def create_data_portal(_bundle_name, _trading_calendar, start_date): global BUNDLE_DATA if not BUNDLE_DATA: set_bundle_data(_bundle_name) # Create a data portal data_portal = DataPortal( BUNDLE_DATA.asset_finder, trading_calendar=_trading_calendar, first_trading_day=start_date, equity_daily_reader=BUNDLE_DATA.equity_daily_bar_reader, adjustment_reader=BUNDLE_DATA.adjustment_reader) return data_portal
def get_window_price(end_date): bundle_name = "custom-csv-bundle" window = 30 # 窗口大小 bundle_data = bundles.load(bundle_name) data_por = DataPortal(bundle_data.asset_finder, get_calendar(calendar_name), bundle_data.equity_daily_bar_reader.first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader) sym = data_por.asset_finder.lookup_symbol(train_symbol_str, end_date) data = data_por.get_history_window(assets=[sym], end_dt=end_date, bar_count=window, frequency='1d', data_frequency='daily', field="close") close = data.iloc[:, 0].values return close
def build_backtest_environment(star_date, end_date,capital_base=100000): trading_calendar = get_calendar("NYSE") sim_params = create_simulation_parameters(capital_base=capital_base, data_frequency='daily', trading_calendar=trading_calendar, start=pd.Timestamp(pd.to_datetime(star_date)).tz_localize('US/Eastern'), end=pd.Timestamp(pd.to_datetime(end_date)).tz_localize('US/Eastern') ) bundle = bundles.load('quandl') prefix, connstr = re.split(r'sqlite:///', str(bundle.asset_finder.engine.url), maxsplit=1, ) env = TradingEnvironment(asset_db_path=connstr, environ=os.environ) data = DataPortal( env.asset_finder, trading_calendar, first_trading_day=bundle.equity_minute_bar_reader.first_trading_day, equity_minute_reader=bundle.equity_minute_bar_reader, equity_daily_reader=bundle.equity_daily_bar_reader, adjustment_reader=bundle.adjustment_reader, ) return data,env,bundle,sim_params
def blotter2bardata(self, equity_minute_reader, blotter): if equity_minute_reader is None: return None dp = DataPortal( asset_finder=self.env.asset_finder, trading_calendar=self.trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader ) restrictions=NoRestrictions() bd = BarData( data_portal=dp, simulation_dt_func=lambda: blotter.current_dt, data_frequency=self.sim_params.data_frequency, trading_calendar=self.trading_calendar, restrictions=restrictions ) return bd
def setUpClass(cls): cls.tempdir = TempDirectory() # asset1 has trades every minute # asset2 has trades every 10 minutes # split_asset trades every minute # illiquid_split_asset trades every 10 minutes cls.env = TradingEnvironment() cls.days = cls.env.days_in_range(start=pd.Timestamp("2016-01-05", tz='UTC'), end=pd.Timestamp("2016-01-07", tz='UTC')) cls.env.write_data( equities_data={ sid: { 'start_date': cls.days[0], 'end_date': cls.days[-1], 'symbol': "ASSET{0}".format(sid) } for sid in [1, 2, 3, 4, 5] }) cls.ASSET1 = cls.env.asset_finder.retrieve_asset(1) cls.ASSET2 = cls.env.asset_finder.retrieve_asset(2) cls.SPLIT_ASSET = cls.env.asset_finder.retrieve_asset(3) cls.ILLIQUID_SPLIT_ASSET = cls.env.asset_finder.retrieve_asset(4) cls.HILARIOUSLY_ILLIQUID_ASSET = cls.env.asset_finder.retrieve_asset(5) cls.ASSETS = [cls.ASSET1, cls.ASSET2] cls.adjustments_reader = cls.create_adjustments_reader() cls.data_portal = DataPortal( cls.env, equity_minute_reader=cls.build_minute_data(), adjustment_reader=cls.adjustments_reader)
def setUpClass(cls): cls.tempdir = TempDirectory() # asset1 has a daily data for each day (1/5, 1/6, 1/7) # asset2 only has daily data for day2 (1/6) cls.env = TradingEnvironment() cls.days = cls.env.days_in_range(start=pd.Timestamp("2016-01-05", tz='UTC'), end=pd.Timestamp("2016-01-08", tz='UTC')) cls.env.write_data( equities_data={ sid: { 'start_date': cls.days[0], 'end_date': cls.days[-1], 'symbol': "ASSET{0}".format(sid) } for sid in [1, 2, 3, 4, 5, 6, 7, 8] }) cls.ASSET1 = cls.env.asset_finder.retrieve_asset(1) cls.ASSET2 = cls.env.asset_finder.retrieve_asset(2) cls.SPLIT_ASSET = cls.env.asset_finder.retrieve_asset(3) cls.ILLIQUID_SPLIT_ASSET = cls.env.asset_finder.retrieve_asset(4) cls.MERGER_ASSET = cls.env.asset_finder.retrieve_asset(5) cls.ILLIQUID_MERGER_ASSET = cls.env.asset_finder.retrieve_asset(6) cls.DIVIDEND_ASSET = cls.env.asset_finder.retrieve_asset(7) cls.ILLIQUID_DIVIDEND_ASSET = cls.env.asset_finder.retrieve_asset(8) cls.ASSETS = [cls.ASSET1, cls.ASSET2] cls.adjustments_reader = cls.create_adjustments_reader() cls.data_portal = DataPortal( cls.env, equity_daily_reader=cls.build_daily_data(), adjustment_reader=cls.adjustments_reader)
def get_zipline_dp(bundle=None, calendar=None): ''' Returns `zipline` data portal, used for symbol lookups and obtaining data. :rtype: `zipline.data.data_portal.DataPortal` :param bundle: optionally specify the `zipline` data bundle to use :param calendar: optionally specify the `zipline` calendar to use :type bundle: `zipline.data.bundles.core.BundleData` :type calendar: `zipline.utils.calendars.exchange_calendar_nyse` type ''' z = _ZIPLINE_QUANDL_BUNDLE bundle = load(z['name']) if bundle is None else bundle calendar = get_calendar(z['cal']) if calendar is None else calendar return DataPortal( bundle.asset_finder, calendar, first_trading_day=bundle.equity_minute_bar_reader.first_trading_day, equity_minute_reader=bundle.equity_minute_bar_reader, equity_daily_reader=bundle.equity_daily_bar_reader, adjustment_reader=bundle.adjustment_reader, )
def create_data_portal_from_trade_history(asset_finder, trading_calendar, tempdir, sim_params, trades_by_sid): if sim_params.data_frequency == "daily": path = os.path.join(tempdir.path, "testdaily.bcolz") writer = BcolzDailyBarWriter( path, trading_calendar, sim_params.start_session, sim_params.end_session ) writer.write( trades_by_sid_to_dfs(trades_by_sid, sim_params.sessions), ) equity_daily_reader = BcolzDailyBarReader(path) return DataPortal( asset_finder, trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, equity_daily_reader=equity_daily_reader, ) else: minutes = trading_calendar.minutes_in_range( sim_params.first_open, sim_params.last_close ) length = len(minutes) assets = {} for sidint, trades in iteritems(trades_by_sid): opens = np.zeros(length) highs = np.zeros(length) lows = np.zeros(length) closes = np.zeros(length) volumes = np.zeros(length) for trade in trades: # put them in the right place idx = minutes.searchsorted(trade.dt) opens[idx] = trade.open_price * 1000 highs[idx] = trade.high * 1000 lows[idx] = trade.low * 1000 closes[idx] = trade.close_price * 1000 volumes[idx] = trade.volume assets[sidint] = pd.DataFrame({ "open": opens, "high": highs, "low": lows, "close": closes, "volume": volumes, "dt": minutes }).set_index("dt") write_bcolz_minute_data( trading_calendar, sim_params.sessions, tempdir.path, assets ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) return DataPortal( asset_finder, trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader, )
def init_instance_fixtures(self): super(TestDataPortal, self).init_instance_fixtures() self.data_portal = DataPortal(self.env, first_trading_day=None)
def init_instance_fixtures(self): super(TestDataPortal, self).init_instance_fixtures() self.data_portal = DataPortal(self.env.asset_finder, self.trading_schedule, first_trading_day=None)
from zipline.utils.calendars import get_calendar from zipline.assets._assets import Future from zipline.utils.run_algo import load_extensions # Load extensions.py; this allows you access to custom bundles load_extensions( default=True, extensions=[], strict=True, environ=os.environ, ) # Set-Up Pricing Data Access trading_calendar = get_calendar('NYSE') bundle = 'futures' bundle_data = bundles.load(bundle) data = DataPortal( bundle_data.asset_finder, trading_calendar=trading_calendar, first_trading_day=bundle_data.equity_daily_bar_reader.first_trading_day, equity_minute_reader=None, equity_daily_reader=bundle_data.equity_daily_bar_reader, future_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) future = bundle_data.asset_finder.lookup_future_symbol continuous_future = bundle_data.asset_finder.create_continuous_future history = data.get_history_window
def transaction_sim(self, **params): """ This is a utility method that asserts expected results for conversion of orders to transactions given a trade history""" tempdir = TempDirectory() try: trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') env = TradingEnvironment() sid = 1 if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( data_frequency="minute") minutes = env.market_minute_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( env, env.days_in_range(minutes[0], minutes[-1]), tempdir.path, assets) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env, equity_minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily") days = sim_params.trading_days assets = { 1: pd.DataFrame( { "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") DailyBarWriterFromDataFrames(assets).write(path, days, assets) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env, equity_daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedSlippage() else: slippage_func = None blotter = Blotter(sim_params.data_frequency, self.env.asset_finder, slippage_func) env.write_data( equities_data={ sid: { "start_date": sim_params.trading_days[0], "end_date": sim_params.trading_days[-1] } }) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = PerformanceTracker(sim_params, self.env) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator**len(order_list) order_id = blotter.order( blotter.asset_finder.retrieve_asset(sid), order_amount * direction, MarketOrder()) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData(data_portal, lambda: tick, sim_params.data_frequency) txns, _ = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) for i in range(order_count): order = order_list[i] self.assertEqual(order.sid, sid) self.assertEqual(order.amount, order_amount * alternator**i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) cumulative_pos = tracker.position_tracker.positions[sid] if total_volume == 0: self.assertIsNone(cumulative_pos) else: self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain sid. oo = blotter.open_orders self.assertNotIn(sid, oo, "Entry is removed when no open orders") finally: tempdir.cleanup()
def choose_loader(column): if column in USEquityPricing.columns: return pipeline_loader raise ValueError("No PipelineLoader registered for column %s." % column) ###################################### data ################################### first_trading_day = \ bundle_data.equity_minute_bar_reader.first_trading_day data = DataPortal( trading_environment.asset_finder, trading_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) ################################## sim_params capital_base = DEFAULT_CAPITAL_BASE start = '2017-1-1' end = '2017-11-30' sim_params = create_simulation_parameters( capital_base=capital_base, start=Date(tz='utc', as_timestamp=True).parser(start), end=Date(tz='utc', as_timestamp=True).parser(end), data_frequency='daily', trading_calendar=trading_calendar, )
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, algo_params, bm_symbol, output, print_algo, local_namespace, environ): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) if bundle is not None: bundle_data = load( bundle, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment(bm_symbol=bm_symbol, asset_db_path=connstr) #env = TradingEnvironment(bm_symbol='000001', # exchange_tz='Asia/Shanghai', # trading_calendar=get_calendar('SHSZ'), # asset_db_path=connstr) first_trading_day =\ bundle_data.equity_minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, get_calendar("SHSZ"), first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) pipeline_loader = USEquityPricingLoader( bundle_data.equity_daily_bar_reader, bundle_data.adjustment_reader, ) def choose_loader(column): if column in USEquityPricing.columns: return pipeline_loader raise ValueError("No PipelineLoader registered for column %s." % column) else: env = TradingEnvironment(environ=environ) choose_loader = None perf = TradingAlgorithm( namespace=namespace, #capital_base=capital_base, algo_params=algo_params, env=env, get_pipeline_loader=choose_loader, sim_params=create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, trading_calendar=get_calendar("SHSZ"), ), **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the zipline magic not write any data perf.to_pickle(output) return perf
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, bundle, bundle_timestamp, custom_data_portal, start, end, output, trading_calendar, print_algo, metrics_set, local_namespace, environ, blotter, benchmark_returns): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. """ if benchmark_returns is None: benchmark_returns, _ = load_market_data(environ=environ) if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) if trading_calendar is None: trading_calendar = get_calendar('XNYS') # date parameter validation if trading_calendar.session_distance(start, end) < 1: raise _RunAlgoError( 'There are no trading days between %s and %s' % ( start.date(), end.date(), ), ) bundle_data = bundles.load( bundle, environ, bundle_timestamp, ) # TODO: Fix this for the custom DataPortal case. first_trading_day = \ bundle_data.equity_minute_bar_reader.first_trading_day if custom_data_portal is None: data = DataPortal( bundle_data.asset_finder, trading_calendar=trading_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) else: data = custom_data_portal # TODO: Fix this for the custom DataPortal case. pipeline_loader = USEquityPricingLoader( bundle_data.equity_daily_bar_reader, bundle_data.adjustment_reader, ) def choose_loader(column): if column in USEquityPricing.columns: return pipeline_loader raise ValueError( "No PipelineLoader registered for column %s." % column ) if isinstance(metrics_set, six.string_types): try: metrics_set = metrics.load(metrics_set) except ValueError as e: raise _RunAlgoError(str(e)) if isinstance(blotter, six.string_types): try: blotter = load(Blotter, blotter) except ValueError as e: raise _RunAlgoError(str(e)) perf = TradingAlgorithm( namespace=namespace, data_portal=data, get_pipeline_loader=choose_loader, trading_calendar=trading_calendar, sim_params=SimulationParameters( start_session=start, end_session=end, trading_calendar=trading_calendar, capital_base=capital_base, data_frequency=data_frequency, ), metrics_set=metrics_set, blotter=blotter, benchmark_returns=benchmark_returns, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, } ).run() if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the zipline magic not write any data perf.to_pickle(output) return perf
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) if bundle is not None: bundle_data = load( bundle, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment(asset_db_path=connstr) data = DataPortal( env, equity_minute_reader=bundle_data.minute_bar_reader, equity_daily_reader=bundle_data.daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) perf = TradingAlgorithm(namespace=namespace, capital_base=capital_base, start=start, end=end, env=env, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': algofile.name, 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the zipline magic not write any data perf.to_pickle(output) return perf
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, bundle, bundle_timestamp, start, end, output, trading_calendar, print_algo, metrics_set, local_namespace, environ, blotter, benchmark_symbol, broker, state_filename): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. additions useful for live trading: broker - wrapper to connect to a real broker state_filename - saving the context of the algo to be able to restart """ log.info("Using bundle '%s'." % bundle) if trading_calendar is None: trading_calendar = get_calendar('XNYS') bundle_data = load_sharadar_bundle(bundle) now = pd.Timestamp.utcnow() if start is None: start = bundle_data.equity_daily_bar_reader.first_trading_day if not broker else now if not trading_calendar.is_session(start.date()): start = trading_calendar.next_open(start) if end is None: end = bundle_data.equity_daily_bar_reader.last_available_dt if not broker else start # date parameter validation if trading_calendar.session_distance(start, end) < 0: raise _RunAlgoError( 'There are no trading days between %s and %s' % ( start.date(), end.date(), ), ) if broker: log.info("Live Trading on %s." % start.date()) else: log.info("Backtest from %s to %s." % (start.date(), end.date())) if benchmark_symbol: benchmark = symbol(benchmark_symbol) benchmark_sid = benchmark.sid benchmark_returns = load_benchmark_data_bundle( bundle_data.equity_daily_bar_reader, benchmark) else: benchmark_sid = None benchmark_returns = pd.Series(index=pd.date_range(start, end, tz='utc'), data=0.0) # emission_rate is a string representing the smallest frequency at which metrics should be reported. # emission_rate will be either minute or daily. When emission_rate is daily, end_of_bar will not be called at all. emission_rate = 'daily' if algotext is not None: if local_namespace: # noinspection PyUnresolvedReferences ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) first_trading_day = \ bundle_data.equity_daily_bar_reader.first_trading_day if isinstance(metrics_set, six.string_types): try: metrics_set = metrics.load(metrics_set) except ValueError as e: raise _RunAlgoError(str(e)) if isinstance(blotter, six.string_types): try: blotter = load(Blotter, blotter) except ValueError as e: raise _RunAlgoError(str(e)) # Special defaults for live trading if broker: data_frequency = 'minute' # No benchmark benchmark_sid = None benchmark_returns = pd.Series(index=pd.date_range(start, end, tz='utc'), data=0.0) broker.daily_bar_reader = bundle_data.equity_daily_bar_reader if start.date() < now.date(): backtest_start = start backtest_end = bundle_data.equity_daily_bar_reader.last_available_dt if not os.path.exists(state_filename): log.info("Backtest from %s to %s." % (backtest_start.date(), backtest_end.date())) backtest_data = DataPortal( bundle_data.asset_finder, trading_calendar=trading_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) backtest = create_algo_class( TradingAlgorithm, backtest_start, backtest_end, algofile, algotext, analyze, before_trading_start, benchmark_returns, benchmark_sid, blotter, bundle_data, capital_base, backtest_data, 'daily', emission_rate, handle_data, initialize, metrics_set, namespace, trading_calendar) ctx_blacklist = ['trading_client'] ctx_whitelist = ['perf_tracker'] ctx_excludes = ctx_blacklist + [ e for e in backtest.__dict__.keys() if e not in ctx_whitelist ] backtest.run() #TODO better logic for the checksumq checksum = getattr(algofile, 'name', '<algorithm>') store_context(state_filename, context=backtest, checksum=checksum, exclude_list=ctx_excludes) else: log.warn("State file already exists. Do not run the backtest.") # Set start and end to now for live trading start = pd.Timestamp.utcnow() if not trading_calendar.is_session(start.date()): start = trading_calendar.next_open(start) end = start # TODO inizia qui per creare un prerun dell'algo prima del live trading # usare store_context prima di passare da TradingAlgorithm a LiveTradingAlgorithm TradingAlgorithmClass = (partial( LiveTradingAlgorithm, broker=broker, state_filename=state_filename) if broker else TradingAlgorithm) DataPortalClass = (partial(DataPortalLive, broker) if broker else DataPortal) data = DataPortalClass( bundle_data.asset_finder, trading_calendar=trading_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) algo = create_algo_class(TradingAlgorithmClass, start, end, algofile, algotext, analyze, before_trading_start, benchmark_returns, benchmark_sid, blotter, bundle_data, capital_base, data, data_frequency, emission_rate, handle_data, initialize, metrics_set, namespace, trading_calendar) perf = algo.run() if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the zipline magic not write any data perf.to_pickle(output) return perf
def init_instance_fixtures(self): super(TestDataPortal, self).init_instance_fixtures() self.data_portal = DataPortal(self.env)