Example #1
0
def create_data_portal(env, tempdir, sim_params, sids, adjustment_reader=None):
    if sim_params.data_frequency == "daily":
        daily_path = write_daily_data(tempdir, sim_params, sids)

        equity_daily_reader = BcolzDailyBarReader(daily_path)

        return DataPortal(
            env,
            equity_daily_reader=equity_daily_reader,
            adjustment_reader=adjustment_reader
        )
    else:
        minutes = env.minutes_for_days_in_range(
            sim_params.first_open,
            sim_params.last_close
        )

        minute_path = write_minute_data(env, tempdir, minutes, sids)

        equity_minute_reader = BcolzMinuteBarReader(minute_path)

        return DataPortal(
            env,
            equity_minute_reader=equity_minute_reader,
            adjustment_reader=adjustment_reader
        )
Example #2
0
def get_prices(numdays=1000):
    now = Timestamp.utcnow()
    bundle = load('quantopian-quandl', os.environ, now)

    all_assets = bundle.asset_finder.retrieve_all(bundle.asset_finder.sids)
    symbols = set(
        str(asset.symbol) for asset in bundle.asset_finder.retrieve_all(
            bundle.asset_finder.equities_sids))

    quandl_data = DataPortal(
        asset_finder=bundle.asset_finder,
        trading_calendar=get_calendar('NYSE'),
        first_trading_day=bundle.equity_daily_bar_reader.first_trading_day,
        equity_minute_reader=bundle.equity_minute_bar_reader,
        equity_daily_reader=bundle.equity_daily_bar_reader,
        adjustment_reader=bundle.adjustment_reader)

    end_dt = pd.Timestamp('2018-03-31', tz='utc')

    quandl_symbols = []
    for ticker in symbols:
        try:
            quandl_symbols.append(
                quandl_data.asset_finder.lookup_symbol(ticker, end_dt))
        except:
            continue
    quandl_pricing = quandl_data.get_history_window(quandl_symbols, end_dt,
                                                    numdays, '1d', 'close',
                                                    'daily')
    quandl_pricing.columns = [c.symbol for c in quandl_pricing.columns]

    return quandl_pricing
def prices(assets, start, end, field='close', offset=0):
    """
    Get price data for assets between start and end.
    """
    start = trading_date(start)
    end = trading_date(end)

    bundle = load_sharadar_bundle()
    trading_calendar = bundle.equity_daily_bar_reader.trading_calendar

    if offset > 0:
        start = trading_calendar.sessions_window(start, -offset)[0]

    bar_count = trading_calendar.session_distance(start, end)

    data_portal = DataPortal(
        bundle.asset_finder,
        trading_calendar=trading_calendar,
        first_trading_day=start,
        equity_daily_reader=bundle.equity_daily_bar_reader,
        adjustment_reader=bundle.adjustment_reader)

    df = data_portal.get_history_window(assets=assets,
                                        end_dt=end,
                                        bar_count=bar_count,
                                        frequency='1d',
                                        field=field,
                                        data_frequency='daily')

    return df if len(assets) > 1 else df.squeeze()
Example #4
0
def create_data_portal(asset_finder, tempdir, sim_params, sids,
                       trading_calendar, adjustment_reader=None):
    if sim_params.data_frequency == "daily":
        daily_path = write_daily_data(tempdir, sim_params, sids,
                                      trading_calendar)

        equity_daily_reader = BcolzDailyBarReader(daily_path)

        return DataPortal(
            asset_finder, trading_calendar,
            first_trading_day=equity_daily_reader.first_trading_day,
            equity_daily_reader=equity_daily_reader,
            adjustment_reader=adjustment_reader
        )
    else:
        minutes = trading_calendar.minutes_in_range(
            sim_params.first_open,
            sim_params.last_close
        )

        minute_path = write_minute_data(trading_calendar, tempdir, minutes,
                                        sids)

        equity_minute_reader = BcolzMinuteBarReader(minute_path)

        return DataPortal(
            asset_finder, trading_calendar,
            first_trading_day=equity_minute_reader.first_trading_day,
            equity_minute_reader=equity_minute_reader,
            adjustment_reader=adjustment_reader
        )
Example #5
0
class TestDataPortal(WithTradingEnvironment, ZiplineTestCase):
    def init_instance_fixtures(self):
        super(TestDataPortal, self).init_instance_fixtures()

        self.data_portal = DataPortal(self.env.asset_finder,
                                      self.trading_schedule,
                                      first_trading_day=None)

    def test_bar_count_for_simple_transforms(self):
        # July 2015
        # Su Mo Tu We Th Fr Sa
        #           1  2  3  4
        #  5  6  7  8  9 10 11
        # 12 13 14 15 16 17 18
        # 19 20 21 22 23 24 25
        # 26 27 28 29 30 31

        # half an hour into july 9, getting a 4-"day" window should get us
        # all the minutes of 7/6, 7/7, 7/8, and 31 minutes of 7/9

        july_9_dt = self.trading_schedule.start_and_end(
            pd.Timestamp("2015-07-09", tz='UTC')
        )[0] + Timedelta("30 minutes")

        self.assertEqual(
            (3 * 390) + 31,
            self.data_portal._get_minute_count_for_transform(july_9_dt, 4)
        )

        #    November 2015
        # Su Mo Tu We Th Fr Sa
        #  1  2  3  4  5  6  7
        #  8  9 10 11 12 13 14
        # 15 16 17 18 19 20 21
        # 22 23 24 25 26 27 28
        # 29 30

        # nov 26th closed
        # nov 27th was an early close

        # half an hour into nov 30, getting a 4-"day" window should get us
        # all the minutes of 11/24, 11/25, 11/27 (half day!), and 31 minutes
        # of 11/30
        nov_30_dt = self.trading_schedule.start_and_end(
            pd.Timestamp("2015-11-30", tz='UTC')
        )[0] + Timedelta("30 minutes")

        self.assertEqual(
            390 + 390 + 210 + 31,
            self.data_portal._get_minute_count_for_transform(nov_30_dt, 4)
        )
Example #6
0
    def test_asset_IPOed_same_day(self):
        # gotta get some minute data up in here.
        # add sid 4 for a couple of days
        minutes = self.trading_calendar.minutes_for_sessions_in_range(
            self.sim_params.sessions[0],
            self.sim_params.sessions[5]
        )

        tmp_reader = tmp_bcolz_equity_minute_bar_reader(
            self.trading_calendar,
            self.trading_calendar.all_sessions,
            create_minute_bar_data(minutes, [2]),
        )
        with tmp_reader as reader:
            data_portal = DataPortal(
                self.env.asset_finder, self.trading_calendar,
                first_trading_day=reader.first_trading_day,
                equity_minute_reader=reader,
                equity_daily_reader=self.bcolz_equity_daily_bar_reader,
                adjustment_reader=self.adjustment_reader,
            )

            source = BenchmarkSource(
                2,
                self.env,
                self.trading_calendar,
                self.sim_params.sessions,
                data_portal
            )

            days_to_use = self.sim_params.sessions

            # first value should be 0.0, coming from daily data
            self.assertAlmostEquals(0.0, source.get_value(days_to_use[0]))

            manually_calculated = data_portal.get_history_window(
                [2], days_to_use[-1],
                len(days_to_use),
                "1d",
                "close",
                "daily",
            )[2].pct_change()

            for idx, day in enumerate(days_to_use[1:]):
                self.assertEqual(
                    source.get_value(day),
                    manually_calculated[idx + 1]
                )
Example #7
0
    def test_orders_stop(self, name, order_data, event_data, expected):
        data = order_data
        data['asset'] = self.ASSET133
        order = Order(**data)

        if expected['transaction']:
            expected['transaction']['asset'] = self.ASSET133
        event_data['asset'] = self.ASSET133

        assets = (
            (133, pd.DataFrame(
                {
                    'open': [event_data['open']],
                    'high': [event_data['high']],
                    'low': [event_data['low']],
                    'close': [event_data['close']],
                    'volume': [event_data['volume']],
                },
                index=[pd.Timestamp('2006-01-05 14:31', tz='UTC')],
            )),
        )
        days = pd.date_range(
            start=normalize_date(self.minutes[0]),
            end=normalize_date(self.minutes[-1])
        )
        with tmp_bcolz_equity_minute_bar_reader(
                self.trading_calendar, days, assets) as reader:
            data_portal = DataPortal(
                self.env.asset_finder, self.trading_calendar,
                first_trading_day=reader.first_trading_day,
                equity_minute_reader=reader,
            )

            slippage_model = VolumeShareSlippage()

            try:
                dt = pd.Timestamp('2006-01-05 14:31', tz='UTC')
                bar_data = BarData(
                    data_portal,
                    lambda: dt,
                    self.sim_params.data_frequency,
                    self.trading_calendar,
                    NoRestrictions(),
                )

                _, txn = next(slippage_model.simulate(
                    bar_data,
                    self.ASSET133,
                    [order],
                ))
            except StopIteration:
                txn = None

            if expected['transaction'] is None:
                self.assertIsNone(txn)
            else:
                self.assertIsNotNone(txn)

                for key, value in expected['transaction'].items():
                    self.assertEquals(value, txn[key])
Example #8
0
    def test_orders_stop(self, name, order_data, event_data, expected):
        tempdir = TempDirectory()
        try:
            data = order_data
            data['sid'] = self.ASSET133

            order = Order(**data)

            assets = {
                133: pd.DataFrame({
                    "open": [event_data["open"]],
                    "high": [event_data["high"]],
                    "low": [event_data["low"]],
                    "close": [event_data["close"]],
                    "volume": [event_data["volume"]],
                    "dt": [pd.Timestamp('2006-01-05 14:31', tz='UTC')]
                }).set_index("dt")
            }

            write_bcolz_minute_data(
                self.env,
                pd.date_range(
                    start=normalize_date(self.minutes[0]),
                    end=normalize_date(self.minutes[-1])
                ),
                tempdir.path,
                assets
            )

            equity_minute_reader = BcolzMinuteBarReader(tempdir.path)

            data_portal = DataPortal(
                self.env,
                equity_minute_reader=equity_minute_reader,
            )

            slippage_model = VolumeShareSlippage()

            try:
                dt = pd.Timestamp('2006-01-05 14:31', tz='UTC')
                bar_data = BarData(data_portal,
                                   lambda: dt,
                                   'minute')
                _, txn = next(slippage_model.simulate(
                    bar_data,
                    self.ASSET133,
                    [order],
                ))
            except StopIteration:
                txn = None

            if expected['transaction'] is None:
                self.assertIsNone(txn)
            else:
                self.assertIsNotNone(txn)

                for key, value in expected['transaction'].items():
                    self.assertEquals(value, txn[key])
        finally:
            tempdir.cleanup()
Example #9
0
def set_bundle(name, calendar='NYSE'):
    global trading_calendar
    global bundle
    global bundle_data
    global engine
    global choose_loader
    global data

    bundle = name
    trading_calendar = get_calendar(calendar)
    bundle_data = bundles.load(bundle)
    engine = SimplePipelineEngine(
        get_loader=choose_loader,
        calendar=trading_calendar.all_sessions,
        asset_finder=bundle_data.asset_finder,
    )

    data = DataPortal(
        bundle_data.asset_finder,
        trading_calendar=trading_calendar,
        first_trading_day=bundle_data.equity_daily_bar_reader.first_trading_day,
        equity_minute_reader=None,
        equity_daily_reader=bundle_data.equity_daily_bar_reader,
        adjustment_reader=bundle_data.adjustment_reader,
    )
Example #10
0
    def test_get_value_during_non_market_hours(self):
        # make sure that if we try to get the OHLCV values of ASSET1 during
        # non-market hours, we don't get the previous market minute's values
        futures_cal = get_calendar("us_futures")

        data_portal = DataPortal(
            self.env.asset_finder,
            futures_cal,
            first_trading_day=self.DATA_PORTAL_FIRST_TRADING_DAY,
            equity_minute_reader=self.bcolz_equity_minute_bar_reader,
        )

        bar_data = BarData(
            data_portal,
            lambda: pd.Timestamp("2016-01-06 3:15", tz="US/Eastern"),
            "minute",
            futures_cal
        )

        self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "open")))
        self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "high")))
        self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "low")))
        self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "close")))
        self.assertEqual(0, bar_data.current(self.ASSET1, "volume"))

        # price should still forward fill
        self.assertEqual(390, bar_data.current(self.ASSET1, "price"))
Example #11
0
def _data_portal(bundle=DEFAULT_BUNDLE):
    """Create a DataPortal for the given bundle.

    Parameters
    ----------
    bundle : str
        The name of the bundle to create a pipeline engine for.

    Returns
    -------
    engine : zipline.pipleine.engine.SimplePipelineEngine
        The pipeline engine which can run pipelines against the bundle.
    calendar : zipline.utils.calendars.TradingCalendar
        The trading calendar for the bundle.
    """
    bundle_data = bundles.load(bundle)
    calendar = bundle_data.equity_daily_bar_reader.trading_calendar
    finder = bundle_data.asset_finder

    data_portal = DataPortal(
        finder,
        calendar,
        calendar.first_session,
        equity_daily_reader=bundle_data.equity_daily_bar_reader,
        adjustment_reader=bundle_data.adjustment_reader)

    return data_portal, calendar
Example #12
0
class TestDataPortal(WithTradingEnvironment, ZiplineTestCase):
    def init_instance_fixtures(self):
        super(TestDataPortal, self).init_instance_fixtures()

        self.data_portal = DataPortal(self.env.asset_finder,
                                      self.trading_schedule,
                                      first_trading_day=None)

    def test_bar_count_for_simple_transforms(self):
        # July 2015
        # Su Mo Tu We Th Fr Sa
        #           1  2  3  4
        #  5  6  7  8  9 10 11
        # 12 13 14 15 16 17 18
        # 19 20 21 22 23 24 25
        # 26 27 28 29 30 31

        # half an hour into july 9, getting a 4-"day" window should get us
        # all the minutes of 7/6, 7/7, 7/8, and 31 minutes of 7/9

        july_9_dt = self.trading_schedule.start_and_end(
            pd.Timestamp("2015-07-09", tz='UTC'))[0] + Timedelta("30 minutes")

        self.assertEqual(
            (3 * 390) + 31,
            self.data_portal._get_minute_count_for_transform(july_9_dt, 4))

        #    November 2015
        # Su Mo Tu We Th Fr Sa
        #  1  2  3  4  5  6  7
        #  8  9 10 11 12 13 14
        # 15 16 17 18 19 20 21
        # 22 23 24 25 26 27 28
        # 29 30

        # nov 26th closed
        # nov 27th was an early close

        # half an hour into nov 30, getting a 4-"day" window should get us
        # all the minutes of 11/24, 11/25, 11/27 (half day!), and 31 minutes
        # of 11/30
        nov_30_dt = self.trading_schedule.start_and_end(
            pd.Timestamp("2015-11-30", tz='UTC'))[0] + Timedelta("30 minutes")

        self.assertEqual(
            390 + 390 + 210 + 31,
            self.data_portal._get_minute_count_for_transform(nov_30_dt, 4))
Example #13
0
    def test_asset_IPOed_same_day(self):
        # gotta get some minute data up in here.
        # add sid 4 for a couple of days
        minutes = self.trading_calendar.minutes_for_sessions_in_range(
            self.sim_params.sessions[0],
            self.sim_params.sessions[5]
        )

        tmp_reader = tmp_bcolz_equity_minute_bar_reader(
            self.trading_calendar,
            self.trading_calendar.all_sessions,
            create_minute_bar_data(minutes, [2]),
        )
        with tmp_reader as reader:
            data_portal = DataPortal(
                self.env.asset_finder, self.trading_calendar,
                first_trading_day=reader.first_trading_day,
                equity_minute_reader=reader,
                equity_daily_reader=self.bcolz_equity_daily_bar_reader,
                adjustment_reader=self.adjustment_reader,
            )

            source = BenchmarkSource(
                2,
                self.env,
                self.trading_calendar,
                self.sim_params.sessions,
                data_portal
            )

            days_to_use = self.sim_params.sessions

            # first value should be 0.0, coming from daily data
            self.assertAlmostEquals(0.0, source.get_value(days_to_use[0]))

            manually_calculated = data_portal.get_history_window(
                [2], days_to_use[-1],
                len(days_to_use),
                "1d",
                "close",
            )[2].pct_change()

            for idx, day in enumerate(days_to_use[1:]):
                self.assertEqual(
                    source.get_value(day),
                    manually_calculated[idx + 1]
                )
Example #14
0
    def setUpClass(cls):
        setup_logger(cls)
        cls.env = trading.TradingEnvironment()

        cls.sim_params = factory.create_simulation_parameters(
            start=pd.Timestamp("2006-01-05", tz='UTC'),
            end=pd.Timestamp("2006-01-06", tz='UTC'))

        cls.env.write_data(
            equities_data={
                24: {
                    'start_date':
                    cls.sim_params.trading_days[0],
                    'end_date':
                    cls.env.next_trading_day(cls.sim_params.trading_days[-1])
                },
                25: {
                    'start_date':
                    cls.sim_params.trading_days[0],
                    'end_date':
                    cls.env.next_trading_day(cls.sim_params.trading_days[-1])
                }
            })

        cls.tempdir = TempDirectory()

        assets = {
            24:
            pd.DataFrame({
                "open": [50, 50],
                "high": [50, 50],
                "low": [50, 50],
                "close": [50, 50],
                "volume": [100, 400],
                "day": [day.value for day in cls.sim_params.trading_days]
            }),
            25:
            pd.DataFrame({
                "open": [50, 50],
                "high": [50, 50],
                "low": [50, 50],
                "close": [50, 50],
                "volume": [100, 400],
                "day": [day.value for day in cls.sim_params.trading_days]
            })
        }

        path = os.path.join(cls.tempdir.path, "tempdata.bcolz")

        DailyBarWriterFromDataFrames(assets).write(path,
                                                   cls.sim_params.trading_days,
                                                   assets)

        equity_daily_reader = BcolzDailyBarReader(path)

        cls.data_portal = DataPortal(
            cls.env,
            equity_daily_reader=equity_daily_reader,
        )
Example #15
0
 def get_data_portal(cls, bundle_data, trading_calendar):
     return DataPortal(
         bundle_data.asset_finder,
         trading_calendar=trading_calendar,
         first_trading_day=bundle_data.equity_daily_bar_reader.
         first_trading_day,
         equity_daily_reader=bundle_data.equity_daily_bar_reader,
         adjustment_reader=bundle_data.adjustment_reader)
Example #16
0
    def setUpClass(cls):
        cls.tempdir = TempDirectory()
        cls.env = TradingEnvironment()

        cls.sim_params = SimulationParameters(
            period_start=pd.Timestamp("2006-01-05 14:31", tz="utc"),
            period_end=pd.Timestamp("2006-01-05 14:36", tz="utc"),
            capital_base=1.0e5,
            data_frequency="minute",
            emission_rate='daily',
            env=cls.env
        )

        cls.sids = [133]

        cls.minutes = pd.DatetimeIndex(
            start=pd.Timestamp("2006-01-05 14:31", tz="utc"),
            end=pd.Timestamp("2006-01-05 14:35", tz="utc"),
            freq="1min"
        )

        assets = {
            133: pd.DataFrame({
                "open": np.array([3.0, 3.0, 3.5, 4.0, 3.5]),
                "high": np.array([3.15, 3.15, 3.15, 3.15, 3.15]),
                "low": np.array([2.85, 2.85, 2.85, 2.85, 2.85]),
                "close": np.array([3.0, 3.5, 4.0, 3.5, 3.0]),
                "volume": [2000, 2000, 2000, 2000, 2000],
                "dt": cls.minutes
            }).set_index("dt")
        }

        write_bcolz_minute_data(
            cls.env,
            pd.date_range(
                start=normalize_date(cls.minutes[0]),
                end=normalize_date(cls.minutes[-1])
            ),
            cls.tempdir.path,
            assets
        )

        cls.env.write_data(equities_data={
            133: {
                "start_date": pd.Timestamp("2006-01-05", tz='utc'),
                "end_date": pd.Timestamp("2006-01-07", tz='utc')
            }
        })

        cls.ASSET133 = cls.env.asset_finder.retrieve_asset(133)

        cls.data_portal = DataPortal(
            cls.env,
            equity_minute_reader=BcolzMinuteBarReader(cls.tempdir.path),
        )
Example #17
0
def create_data_portal(_bundle_name, _trading_calendar, start_date):
    global BUNDLE_DATA
    if not BUNDLE_DATA:
        set_bundle_data(_bundle_name)
    # Create a data portal
    data_portal = DataPortal(
        BUNDLE_DATA.asset_finder,
        trading_calendar=_trading_calendar,
        first_trading_day=start_date,
        equity_daily_reader=BUNDLE_DATA.equity_daily_bar_reader,
        adjustment_reader=BUNDLE_DATA.adjustment_reader)
    return data_portal
def get_window_price(end_date):
    bundle_name = "custom-csv-bundle"
    window = 30  # 窗口大小

    bundle_data = bundles.load(bundle_name)
    data_por = DataPortal(bundle_data.asset_finder,
                          get_calendar(calendar_name),
                          bundle_data.equity_daily_bar_reader.first_trading_day,
                          equity_minute_reader=bundle_data.equity_minute_bar_reader,
                          equity_daily_reader=bundle_data.equity_daily_bar_reader,
                          adjustment_reader=bundle_data.adjustment_reader)

    sym = data_por.asset_finder.lookup_symbol(train_symbol_str, end_date)
    data = data_por.get_history_window(assets=[sym],
                                       end_dt=end_date,
                                       bar_count=window,
                                       frequency='1d',
                                       data_frequency='daily',
                                       field="close")

    close = data.iloc[:, 0].values
    return close
Example #19
0
def build_backtest_environment(star_date, end_date,capital_base=100000):
    trading_calendar = get_calendar("NYSE")
    sim_params = create_simulation_parameters(capital_base=capital_base,
                                              data_frequency='daily',
                                              trading_calendar=trading_calendar,
                                              start=pd.Timestamp(pd.to_datetime(star_date)).tz_localize('US/Eastern'),
                                              end=pd.Timestamp(pd.to_datetime(end_date)).tz_localize('US/Eastern')
                                              )
    bundle = bundles.load('quandl')
    prefix, connstr = re.split(r'sqlite:///', str(bundle.asset_finder.engine.url), maxsplit=1, )
    env = TradingEnvironment(asset_db_path=connstr, environ=os.environ)
    data = DataPortal(
        env.asset_finder, trading_calendar,
        first_trading_day=bundle.equity_minute_bar_reader.first_trading_day,
        equity_minute_reader=bundle.equity_minute_bar_reader,
        equity_daily_reader=bundle.equity_daily_bar_reader,
        adjustment_reader=bundle.adjustment_reader,
    )
    return data,env,bundle,sim_params
Example #20
0
  def blotter2bardata(self, equity_minute_reader, blotter):
    if equity_minute_reader is None:
      return None

    dp = DataPortal(
      asset_finder=self.env.asset_finder,
      trading_calendar=self.trading_calendar,
      first_trading_day=equity_minute_reader.first_trading_day,
      equity_minute_reader=equity_minute_reader
    )

    restrictions=NoRestrictions()

    bd = BarData(
      data_portal=dp,
      simulation_dt_func=lambda: blotter.current_dt,
      data_frequency=self.sim_params.data_frequency,
      trading_calendar=self.trading_calendar,
      restrictions=restrictions
    )

    return bd
Example #21
0
    def setUpClass(cls):
        cls.tempdir = TempDirectory()

        # asset1 has trades every minute
        # asset2 has trades every 10 minutes
        # split_asset trades every minute
        # illiquid_split_asset trades every 10 minutes

        cls.env = TradingEnvironment()

        cls.days = cls.env.days_in_range(start=pd.Timestamp("2016-01-05",
                                                            tz='UTC'),
                                         end=pd.Timestamp("2016-01-07",
                                                          tz='UTC'))

        cls.env.write_data(
            equities_data={
                sid: {
                    'start_date': cls.days[0],
                    'end_date': cls.days[-1],
                    'symbol': "ASSET{0}".format(sid)
                }
                for sid in [1, 2, 3, 4, 5]
            })

        cls.ASSET1 = cls.env.asset_finder.retrieve_asset(1)
        cls.ASSET2 = cls.env.asset_finder.retrieve_asset(2)
        cls.SPLIT_ASSET = cls.env.asset_finder.retrieve_asset(3)
        cls.ILLIQUID_SPLIT_ASSET = cls.env.asset_finder.retrieve_asset(4)
        cls.HILARIOUSLY_ILLIQUID_ASSET = cls.env.asset_finder.retrieve_asset(5)

        cls.ASSETS = [cls.ASSET1, cls.ASSET2]

        cls.adjustments_reader = cls.create_adjustments_reader()
        cls.data_portal = DataPortal(
            cls.env,
            equity_minute_reader=cls.build_minute_data(),
            adjustment_reader=cls.adjustments_reader)
Example #22
0
    def setUpClass(cls):
        cls.tempdir = TempDirectory()

        # asset1 has a daily data for each day (1/5, 1/6, 1/7)
        # asset2 only has daily data for day2 (1/6)

        cls.env = TradingEnvironment()

        cls.days = cls.env.days_in_range(start=pd.Timestamp("2016-01-05",
                                                            tz='UTC'),
                                         end=pd.Timestamp("2016-01-08",
                                                          tz='UTC'))

        cls.env.write_data(
            equities_data={
                sid: {
                    'start_date': cls.days[0],
                    'end_date': cls.days[-1],
                    'symbol': "ASSET{0}".format(sid)
                }
                for sid in [1, 2, 3, 4, 5, 6, 7, 8]
            })

        cls.ASSET1 = cls.env.asset_finder.retrieve_asset(1)
        cls.ASSET2 = cls.env.asset_finder.retrieve_asset(2)
        cls.SPLIT_ASSET = cls.env.asset_finder.retrieve_asset(3)
        cls.ILLIQUID_SPLIT_ASSET = cls.env.asset_finder.retrieve_asset(4)
        cls.MERGER_ASSET = cls.env.asset_finder.retrieve_asset(5)
        cls.ILLIQUID_MERGER_ASSET = cls.env.asset_finder.retrieve_asset(6)
        cls.DIVIDEND_ASSET = cls.env.asset_finder.retrieve_asset(7)
        cls.ILLIQUID_DIVIDEND_ASSET = cls.env.asset_finder.retrieve_asset(8)
        cls.ASSETS = [cls.ASSET1, cls.ASSET2]

        cls.adjustments_reader = cls.create_adjustments_reader()
        cls.data_portal = DataPortal(
            cls.env,
            equity_daily_reader=cls.build_daily_data(),
            adjustment_reader=cls.adjustments_reader)
Example #23
0
def get_zipline_dp(bundle=None, calendar=None):
    '''
    Returns `zipline` data portal, used for symbol lookups and obtaining data.

    :rtype: `zipline.data.data_portal.DataPortal`
    :param bundle: optionally specify the `zipline` data bundle to use
    :param calendar: optionally specify the `zipline` calendar to use
    :type bundle: `zipline.data.bundles.core.BundleData`
    :type calendar: `zipline.utils.calendars.exchange_calendar_nyse` type
    '''

    z = _ZIPLINE_QUANDL_BUNDLE
    bundle = load(z['name']) if bundle is None else bundle
    calendar = get_calendar(z['cal']) if calendar is None else calendar

    return DataPortal(
        bundle.asset_finder,
        calendar,
        first_trading_day=bundle.equity_minute_bar_reader.first_trading_day,
        equity_minute_reader=bundle.equity_minute_bar_reader,
        equity_daily_reader=bundle.equity_daily_bar_reader,
        adjustment_reader=bundle.adjustment_reader,
    )
Example #24
0
def create_data_portal_from_trade_history(asset_finder, trading_calendar,
                                          tempdir, sim_params, trades_by_sid):
    if sim_params.data_frequency == "daily":
        path = os.path.join(tempdir.path, "testdaily.bcolz")
        writer = BcolzDailyBarWriter(
            path, trading_calendar,
            sim_params.start_session,
            sim_params.end_session
        )
        writer.write(
            trades_by_sid_to_dfs(trades_by_sid, sim_params.sessions),
        )

        equity_daily_reader = BcolzDailyBarReader(path)

        return DataPortal(
            asset_finder, trading_calendar,
            first_trading_day=equity_daily_reader.first_trading_day,
            equity_daily_reader=equity_daily_reader,
        )
    else:
        minutes = trading_calendar.minutes_in_range(
            sim_params.first_open,
            sim_params.last_close
        )

        length = len(minutes)
        assets = {}

        for sidint, trades in iteritems(trades_by_sid):
            opens = np.zeros(length)
            highs = np.zeros(length)
            lows = np.zeros(length)
            closes = np.zeros(length)
            volumes = np.zeros(length)

            for trade in trades:
                # put them in the right place
                idx = minutes.searchsorted(trade.dt)

                opens[idx] = trade.open_price * 1000
                highs[idx] = trade.high * 1000
                lows[idx] = trade.low * 1000
                closes[idx] = trade.close_price * 1000
                volumes[idx] = trade.volume

            assets[sidint] = pd.DataFrame({
                "open": opens,
                "high": highs,
                "low": lows,
                "close": closes,
                "volume": volumes,
                "dt": minutes
            }).set_index("dt")

        write_bcolz_minute_data(
            trading_calendar,
            sim_params.sessions,
            tempdir.path,
            assets
        )

        equity_minute_reader = BcolzMinuteBarReader(tempdir.path)

        return DataPortal(
            asset_finder, trading_calendar,
            first_trading_day=equity_minute_reader.first_trading_day,
            equity_minute_reader=equity_minute_reader,
        )
Example #25
0
    def init_instance_fixtures(self):
        super(TestDataPortal, self).init_instance_fixtures()

        self.data_portal = DataPortal(self.env, first_trading_day=None)
Example #26
0
    def init_instance_fixtures(self):
        super(TestDataPortal, self).init_instance_fixtures()

        self.data_portal = DataPortal(self.env.asset_finder,
                                      self.trading_schedule,
                                      first_trading_day=None)
Example #27
0
from zipline.utils.calendars import get_calendar
from zipline.assets._assets import Future
from zipline.utils.run_algo import load_extensions

# Load extensions.py; this allows you access to custom bundles
load_extensions(
    default=True,
    extensions=[],
    strict=True,
    environ=os.environ,
)

# Set-Up Pricing Data Access
trading_calendar = get_calendar('NYSE')
bundle = 'futures'
bundle_data = bundles.load(bundle)

data = DataPortal(
    bundle_data.asset_finder,
    trading_calendar=trading_calendar,
    first_trading_day=bundle_data.equity_daily_bar_reader.first_trading_day,
    equity_minute_reader=None,
    equity_daily_reader=bundle_data.equity_daily_bar_reader,
    future_daily_reader=bundle_data.equity_daily_bar_reader,
    adjustment_reader=bundle_data.adjustment_reader,
)

future = bundle_data.asset_finder.lookup_future_symbol
continuous_future = bundle_data.asset_finder.create_continuous_future
history = data.get_history_window
Example #28
0
    def transaction_sim(self, **params):
        """ This is a utility method that asserts expected
        results for conversion of orders to transactions given a
        trade history"""
        tempdir = TempDirectory()
        try:
            trade_count = params['trade_count']
            trade_interval = params['trade_interval']
            order_count = params['order_count']
            order_amount = params['order_amount']
            order_interval = params['order_interval']
            expected_txn_count = params['expected_txn_count']
            expected_txn_volume = params['expected_txn_volume']

            # optional parameters
            # ---------------------
            # if present, alternate between long and short sales
            alternate = params.get('alternate')

            # if present, expect transaction amounts to match orders exactly.
            complete_fill = params.get('complete_fill')

            env = TradingEnvironment()

            sid = 1

            if trade_interval < timedelta(days=1):
                sim_params = factory.create_simulation_parameters(
                    data_frequency="minute")

                minutes = env.market_minute_window(
                    sim_params.first_open,
                    int((trade_interval.total_seconds() / 60) * trade_count) +
                    100)

                price_data = np.array([10.1] * len(minutes))
                assets = {
                    sid:
                    pd.DataFrame({
                        "open": price_data,
                        "high": price_data,
                        "low": price_data,
                        "close": price_data,
                        "volume": np.array([100] * len(minutes)),
                        "dt": minutes
                    }).set_index("dt")
                }

                write_bcolz_minute_data(
                    env, env.days_in_range(minutes[0], minutes[-1]),
                    tempdir.path, assets)

                equity_minute_reader = BcolzMinuteBarReader(tempdir.path)

                data_portal = DataPortal(
                    env,
                    equity_minute_reader=equity_minute_reader,
                )
            else:
                sim_params = factory.create_simulation_parameters(
                    data_frequency="daily")

                days = sim_params.trading_days

                assets = {
                    1:
                    pd.DataFrame(
                        {
                            "open": [10.1] * len(days),
                            "high": [10.1] * len(days),
                            "low": [10.1] * len(days),
                            "close": [10.1] * len(days),
                            "volume": [100] * len(days),
                            "day": [day.value for day in days]
                        },
                        index=days)
                }

                path = os.path.join(tempdir.path, "testdata.bcolz")
                DailyBarWriterFromDataFrames(assets).write(path, days, assets)

                equity_daily_reader = BcolzDailyBarReader(path)

                data_portal = DataPortal(
                    env,
                    equity_daily_reader=equity_daily_reader,
                )

            if "default_slippage" not in params or \
               not params["default_slippage"]:
                slippage_func = FixedSlippage()
            else:
                slippage_func = None

            blotter = Blotter(sim_params.data_frequency, self.env.asset_finder,
                              slippage_func)

            env.write_data(
                equities_data={
                    sid: {
                        "start_date": sim_params.trading_days[0],
                        "end_date": sim_params.trading_days[-1]
                    }
                })

            start_date = sim_params.first_open

            if alternate:
                alternator = -1
            else:
                alternator = 1

            tracker = PerformanceTracker(sim_params, self.env)

            # replicate what tradesim does by going through every minute or day
            # of the simulation and processing open orders each time
            if sim_params.data_frequency == "minute":
                ticks = minutes
            else:
                ticks = days

            transactions = []

            order_list = []
            order_date = start_date
            for tick in ticks:
                blotter.current_dt = tick
                if tick >= order_date and len(order_list) < order_count:
                    # place an order
                    direction = alternator**len(order_list)
                    order_id = blotter.order(
                        blotter.asset_finder.retrieve_asset(sid),
                        order_amount * direction, MarketOrder())
                    order_list.append(blotter.orders[order_id])
                    order_date = order_date + order_interval
                    # move after market orders to just after market next
                    # market open.
                    if order_date.hour >= 21:
                        if order_date.minute >= 00:
                            order_date = order_date + timedelta(days=1)
                            order_date = order_date.replace(hour=14, minute=30)
                else:
                    bar_data = BarData(data_portal, lambda: tick,
                                       sim_params.data_frequency)
                    txns, _ = blotter.get_transactions(bar_data)
                    for txn in txns:
                        tracker.process_transaction(txn)
                        transactions.append(txn)

            for i in range(order_count):
                order = order_list[i]
                self.assertEqual(order.sid, sid)
                self.assertEqual(order.amount, order_amount * alternator**i)

            if complete_fill:
                self.assertEqual(len(transactions), len(order_list))

            total_volume = 0
            for i in range(len(transactions)):
                txn = transactions[i]
                total_volume += txn.amount
                if complete_fill:
                    order = order_list[i]
                    self.assertEqual(order.amount, txn.amount)

            self.assertEqual(total_volume, expected_txn_volume)

            self.assertEqual(len(transactions), expected_txn_count)

            cumulative_pos = tracker.position_tracker.positions[sid]
            if total_volume == 0:
                self.assertIsNone(cumulative_pos)
            else:
                self.assertEqual(total_volume, cumulative_pos.amount)

            # the open orders should not contain sid.
            oo = blotter.open_orders
            self.assertNotIn(sid, oo, "Entry is removed when no open orders")
        finally:
            tempdir.cleanup()
Example #29
0
    def init_instance_fixtures(self):
        super(TestDataPortal, self).init_instance_fixtures()

        self.data_portal = DataPortal(self.env.asset_finder,
                                      self.trading_schedule,
                                      first_trading_day=None)
Example #30
0

def choose_loader(column):
    if column in USEquityPricing.columns:
        return pipeline_loader
    raise ValueError("No PipelineLoader registered for column %s." % column)


###################################### data ###################################

first_trading_day = \
    bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
    trading_environment.asset_finder,
    trading_calendar,
    first_trading_day=first_trading_day,
    equity_minute_reader=bundle_data.equity_minute_bar_reader,
    equity_daily_reader=bundle_data.equity_daily_bar_reader,
    adjustment_reader=bundle_data.adjustment_reader,
)
################################## sim_params
capital_base = DEFAULT_CAPITAL_BASE
start = '2017-1-1'
end = '2017-11-30'
sim_params = create_simulation_parameters(
    capital_base=capital_base,
    start=Date(tz='utc', as_timestamp=True).parser(start),
    end=Date(tz='utc', as_timestamp=True).parser(end),
    data_frequency='daily',
    trading_calendar=trading_calendar,
)
Example #31
0
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
         algotext, defines, data_frequency, capital_base, data, bundle,
         bundle_timestamp, start, end, algo_params, bm_symbol, output,
         print_algo, local_namespace, environ):
    """Run a backtest for the given algorithm.

    This is shared between the cli and :func:`zipline.run_algo`.
    """
    if algotext is not None:
        if local_namespace:
            ip = get_ipython()  # noqa
            namespace = ip.user_ns
        else:
            namespace = {}

        for assign in defines:
            try:
                name, value = assign.split('=', 2)
            except ValueError:
                raise ValueError(
                    'invalid define %r, should be of the form name=value' %
                    assign, )
            try:
                # evaluate in the same namespace so names may refer to
                # eachother
                namespace[name] = eval(value, namespace)
            except Exception as e:
                raise ValueError(
                    'failed to execute definition for name %r: %s' %
                    (name, e), )
    elif defines:
        raise _RunAlgoError(
            'cannot pass define without `algotext`',
            "cannot pass '-D' / '--define' without '-t' / '--algotext'",
        )
    else:
        namespace = {}
        if algofile is not None:
            algotext = algofile.read()

    if print_algo:
        if PYGMENTS:
            highlight(
                algotext,
                PythonLexer(),
                TerminalFormatter(),
                outfile=sys.stdout,
            )
        else:
            click.echo(algotext)

    if bundle is not None:
        bundle_data = load(
            bundle,
            environ,
            bundle_timestamp,
        )

        prefix, connstr = re.split(
            r'sqlite:///',
            str(bundle_data.asset_finder.engine.url),
            maxsplit=1,
        )
        if prefix:
            raise ValueError(
                "invalid url %r, must begin with 'sqlite:///'" %
                str(bundle_data.asset_finder.engine.url), )
        env = TradingEnvironment(bm_symbol=bm_symbol, asset_db_path=connstr)
        #env = TradingEnvironment(bm_symbol='000001',
        #                         exchange_tz='Asia/Shanghai',
        #                         trading_calendar=get_calendar('SHSZ'),
        #                         asset_db_path=connstr)
        first_trading_day =\
            bundle_data.equity_minute_bar_reader.first_trading_day
        data = DataPortal(
            env.asset_finder,
            get_calendar("SHSZ"),
            first_trading_day=first_trading_day,
            equity_minute_reader=bundle_data.equity_minute_bar_reader,
            equity_daily_reader=bundle_data.equity_daily_bar_reader,
            adjustment_reader=bundle_data.adjustment_reader,
        )

        pipeline_loader = USEquityPricingLoader(
            bundle_data.equity_daily_bar_reader,
            bundle_data.adjustment_reader,
        )

        def choose_loader(column):
            if column in USEquityPricing.columns:
                return pipeline_loader
            raise ValueError("No PipelineLoader registered for column %s." %
                             column)
    else:
        env = TradingEnvironment(environ=environ)
        choose_loader = None

    perf = TradingAlgorithm(
        namespace=namespace,
        #capital_base=capital_base,
        algo_params=algo_params,
        env=env,
        get_pipeline_loader=choose_loader,
        sim_params=create_simulation_parameters(
            start=start,
            end=end,
            capital_base=capital_base,
            data_frequency=data_frequency,
            trading_calendar=get_calendar("SHSZ"),
        ),
        **{
            'initialize': initialize,
            'handle_data': handle_data,
            'before_trading_start': before_trading_start,
            'analyze': analyze,
        } if algotext is None else {
            'algo_filename': getattr(algofile, 'name', '<algorithm>'),
            'script': algotext,
        }).run(
            data,
            overwrite_sim_params=False,
        )

    if output == '-':
        click.echo(str(perf))
    elif output != os.devnull:  # make the zipline magic not write any data
        perf.to_pickle(output)

    return perf
Example #32
0
def _run(handle_data,
         initialize,
         before_trading_start,
         analyze,
         algofile,
         algotext,
         defines,
         data_frequency,
         capital_base,
         bundle,
         bundle_timestamp,
         custom_data_portal,
         start,
         end,
         output,
         trading_calendar,
         print_algo,
         metrics_set,
         local_namespace,
         environ,
         blotter,
         benchmark_returns):
    """Run a backtest for the given algorithm.

    This is shared between the cli and :func:`zipline.run_algo`.
    """
    if benchmark_returns is None:
        benchmark_returns, _ = load_market_data(environ=environ)

    if algotext is not None:
        if local_namespace:
            ip = get_ipython()  # noqa
            namespace = ip.user_ns
        else:
            namespace = {}

        for assign in defines:
            try:
                name, value = assign.split('=', 2)
            except ValueError:
                raise ValueError(
                    'invalid define %r, should be of the form name=value' %
                    assign,
                )
            try:
                # evaluate in the same namespace so names may refer to
                # eachother
                namespace[name] = eval(value, namespace)
            except Exception as e:
                raise ValueError(
                    'failed to execute definition for name %r: %s' % (name, e),
                )
    elif defines:
        raise _RunAlgoError(
            'cannot pass define without `algotext`',
            "cannot pass '-D' / '--define' without '-t' / '--algotext'",
        )
    else:
        namespace = {}
        if algofile is not None:
            algotext = algofile.read()

    if print_algo:
        if PYGMENTS:
            highlight(
                algotext,
                PythonLexer(),
                TerminalFormatter(),
                outfile=sys.stdout,
            )
        else:
            click.echo(algotext)

    if trading_calendar is None:
        trading_calendar = get_calendar('XNYS')

    # date parameter validation
    if trading_calendar.session_distance(start, end) < 1:
        raise _RunAlgoError(
            'There are no trading days between %s and %s' % (
                start.date(),
                end.date(),
            ),
        )

    bundle_data = bundles.load(
        bundle,
        environ,
        bundle_timestamp,
    )

    # TODO: Fix this for the custom DataPortal case.
    first_trading_day = \
        bundle_data.equity_minute_bar_reader.first_trading_day

    if custom_data_portal is None:
        data = DataPortal(
            bundle_data.asset_finder,
            trading_calendar=trading_calendar,
            first_trading_day=first_trading_day,
            equity_minute_reader=bundle_data.equity_minute_bar_reader,
            equity_daily_reader=bundle_data.equity_daily_bar_reader,
            adjustment_reader=bundle_data.adjustment_reader,
        )
    else:
        data = custom_data_portal

    # TODO: Fix this for the custom DataPortal case.
    pipeline_loader = USEquityPricingLoader(
        bundle_data.equity_daily_bar_reader,
        bundle_data.adjustment_reader,
    )

    def choose_loader(column):
        if column in USEquityPricing.columns:
            return pipeline_loader
        raise ValueError(
            "No PipelineLoader registered for column %s." % column
        )

    if isinstance(metrics_set, six.string_types):
        try:
            metrics_set = metrics.load(metrics_set)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    if isinstance(blotter, six.string_types):
        try:
            blotter = load(Blotter, blotter)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    perf = TradingAlgorithm(
        namespace=namespace,
        data_portal=data,
        get_pipeline_loader=choose_loader,
        trading_calendar=trading_calendar,
        sim_params=SimulationParameters(
            start_session=start,
            end_session=end,
            trading_calendar=trading_calendar,
            capital_base=capital_base,
            data_frequency=data_frequency,
        ),
        metrics_set=metrics_set,
        blotter=blotter,
        benchmark_returns=benchmark_returns,
        **{
            'initialize': initialize,
            'handle_data': handle_data,
            'before_trading_start': before_trading_start,
            'analyze': analyze,
        } if algotext is None else {
            'algo_filename': getattr(algofile, 'name', '<algorithm>'),
            'script': algotext,
        }
    ).run()

    if output == '-':
        click.echo(str(perf))
    elif output != os.devnull:  # make the zipline magic not write any data
        perf.to_pickle(output)

    return perf
Example #33
0
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
         algotext, defines, data_frequency, capital_base, data, bundle,
         bundle_timestamp, start, end, output, print_algo, local_namespace,
         environ):
    """Run a backtest for the given algorithm.

    This is shared between the cli and :func:`zipline.run_algo`.
    """
    if algotext is not None:
        if local_namespace:
            ip = get_ipython()  # noqa
            namespace = ip.user_ns
        else:
            namespace = {}

        for assign in defines:
            try:
                name, value = assign.split('=', 2)
            except ValueError:
                raise ValueError(
                    'invalid define %r, should be of the form name=value' %
                    assign, )
            try:
                # evaluate in the same namespace so names may refer to
                # eachother
                namespace[name] = eval(value, namespace)
            except Exception as e:
                raise ValueError(
                    'failed to execute definition for name %r: %s' %
                    (name, e), )
    elif defines:
        raise _RunAlgoError(
            'cannot pass define without `algotext`',
            "cannot pass '-D' / '--define' without '-t' / '--algotext'",
        )
    else:
        namespace = {}
        if algofile is not None:
            algotext = algofile.read()

    if print_algo:
        if PYGMENTS:
            highlight(
                algotext,
                PythonLexer(),
                TerminalFormatter(),
                outfile=sys.stdout,
            )
        else:
            click.echo(algotext)

    if bundle is not None:
        bundle_data = load(
            bundle,
            environ,
            bundle_timestamp,
        )

        prefix, connstr = re.split(
            r'sqlite:///',
            str(bundle_data.asset_finder.engine.url),
            maxsplit=1,
        )
        if prefix:
            raise ValueError(
                "invalid url %r, must begin with 'sqlite:///'" %
                str(bundle_data.asset_finder.engine.url), )
        env = TradingEnvironment(asset_db_path=connstr)
        data = DataPortal(
            env,
            equity_minute_reader=bundle_data.minute_bar_reader,
            equity_daily_reader=bundle_data.daily_bar_reader,
            adjustment_reader=bundle_data.adjustment_reader,
        )

    perf = TradingAlgorithm(namespace=namespace,
                            capital_base=capital_base,
                            start=start,
                            end=end,
                            env=env,
                            **{
                                'initialize': initialize,
                                'handle_data': handle_data,
                                'before_trading_start': before_trading_start,
                                'analyze': analyze,
                            } if algotext is None else {
                                'algo_filename': algofile.name,
                                'script': algotext,
                            }).run(
                                data,
                                overwrite_sim_params=False,
                            )

    if output == '-':
        click.echo(str(perf))
    elif output != os.devnull:  # make the zipline magic not write any data
        perf.to_pickle(output)

    return perf
Example #34
0
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
         algotext, defines, data_frequency, capital_base, bundle,
         bundle_timestamp, start, end, output, trading_calendar, print_algo,
         metrics_set, local_namespace, environ, blotter, benchmark_symbol,
         broker, state_filename):
    """Run a backtest for the given algorithm.

    This is shared between the cli and :func:`zipline.run_algo`.

    additions useful for live trading:
    broker - wrapper to connect to a real broker
    state_filename - saving the context of the algo to be able to restart
    """
    log.info("Using bundle '%s'." % bundle)

    if trading_calendar is None:
        trading_calendar = get_calendar('XNYS')

    bundle_data = load_sharadar_bundle(bundle)
    now = pd.Timestamp.utcnow()
    if start is None:
        start = bundle_data.equity_daily_bar_reader.first_trading_day if not broker else now

    if not trading_calendar.is_session(start.date()):
        start = trading_calendar.next_open(start)

    if end is None:
        end = bundle_data.equity_daily_bar_reader.last_available_dt if not broker else start

    # date parameter validation
    if trading_calendar.session_distance(start, end) < 0:
        raise _RunAlgoError(
            'There are no trading days between %s and %s' % (
                start.date(),
                end.date(),
            ), )

    if broker:
        log.info("Live Trading on %s." % start.date())
    else:
        log.info("Backtest from %s to %s." % (start.date(), end.date()))

    if benchmark_symbol:
        benchmark = symbol(benchmark_symbol)
        benchmark_sid = benchmark.sid
        benchmark_returns = load_benchmark_data_bundle(
            bundle_data.equity_daily_bar_reader, benchmark)
    else:
        benchmark_sid = None
        benchmark_returns = pd.Series(index=pd.date_range(start, end,
                                                          tz='utc'),
                                      data=0.0)

    # emission_rate is a string representing the smallest frequency at which metrics should be reported.
    # emission_rate will be either minute or daily. When emission_rate is daily, end_of_bar will not be called at all.
    emission_rate = 'daily'

    if algotext is not None:
        if local_namespace:
            # noinspection PyUnresolvedReferences
            ip = get_ipython()  # noqa
            namespace = ip.user_ns
        else:
            namespace = {}

        for assign in defines:
            try:
                name, value = assign.split('=', 2)
            except ValueError:
                raise ValueError(
                    'invalid define %r, should be of the form name=value' %
                    assign, )
            try:
                # evaluate in the same namespace so names may refer to
                # eachother
                namespace[name] = eval(value, namespace)
            except Exception as e:
                raise ValueError(
                    'failed to execute definition for name %r: %s' %
                    (name, e), )
    elif defines:
        raise _RunAlgoError(
            'cannot pass define without `algotext`',
            "cannot pass '-D' / '--define' without '-t' / '--algotext'",
        )
    else:
        namespace = {}
        if algofile is not None:
            algotext = algofile.read()

    if print_algo:
        if PYGMENTS:
            highlight(
                algotext,
                PythonLexer(),
                TerminalFormatter(),
                outfile=sys.stdout,
            )
        else:
            click.echo(algotext)

    first_trading_day = \
        bundle_data.equity_daily_bar_reader.first_trading_day

    if isinstance(metrics_set, six.string_types):
        try:
            metrics_set = metrics.load(metrics_set)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    if isinstance(blotter, six.string_types):
        try:
            blotter = load(Blotter, blotter)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    # Special defaults for live trading
    if broker:
        data_frequency = 'minute'

        # No benchmark
        benchmark_sid = None
        benchmark_returns = pd.Series(index=pd.date_range(start, end,
                                                          tz='utc'),
                                      data=0.0)

        broker.daily_bar_reader = bundle_data.equity_daily_bar_reader

        if start.date() < now.date():
            backtest_start = start
            backtest_end = bundle_data.equity_daily_bar_reader.last_available_dt

            if not os.path.exists(state_filename):
                log.info("Backtest from %s to %s." %
                         (backtest_start.date(), backtest_end.date()))
                backtest_data = DataPortal(
                    bundle_data.asset_finder,
                    trading_calendar=trading_calendar,
                    first_trading_day=first_trading_day,
                    equity_minute_reader=bundle_data.equity_minute_bar_reader,
                    equity_daily_reader=bundle_data.equity_daily_bar_reader,
                    adjustment_reader=bundle_data.adjustment_reader,
                )
                backtest = create_algo_class(
                    TradingAlgorithm, backtest_start, backtest_end, algofile,
                    algotext, analyze, before_trading_start, benchmark_returns,
                    benchmark_sid, blotter, bundle_data, capital_base,
                    backtest_data, 'daily', emission_rate, handle_data,
                    initialize, metrics_set, namespace, trading_calendar)

                ctx_blacklist = ['trading_client']
                ctx_whitelist = ['perf_tracker']
                ctx_excludes = ctx_blacklist + [
                    e
                    for e in backtest.__dict__.keys() if e not in ctx_whitelist
                ]
                backtest.run()
                #TODO better logic for the checksumq
                checksum = getattr(algofile, 'name', '<algorithm>')
                store_context(state_filename,
                              context=backtest,
                              checksum=checksum,
                              exclude_list=ctx_excludes)
            else:
                log.warn("State file already exists. Do not run the backtest.")

            # Set start and end to now for live trading
            start = pd.Timestamp.utcnow()
            if not trading_calendar.is_session(start.date()):
                start = trading_calendar.next_open(start)
            end = start

    # TODO inizia qui per creare un prerun dell'algo prima del live trading
    # usare store_context prima di passare da TradingAlgorithm a LiveTradingAlgorithm
    TradingAlgorithmClass = (partial(
        LiveTradingAlgorithm, broker=broker, state_filename=state_filename)
                             if broker else TradingAlgorithm)

    DataPortalClass = (partial(DataPortalLive, broker)
                       if broker else DataPortal)
    data = DataPortalClass(
        bundle_data.asset_finder,
        trading_calendar=trading_calendar,
        first_trading_day=first_trading_day,
        equity_minute_reader=bundle_data.equity_minute_bar_reader,
        equity_daily_reader=bundle_data.equity_daily_bar_reader,
        adjustment_reader=bundle_data.adjustment_reader,
    )
    algo = create_algo_class(TradingAlgorithmClass, start, end, algofile,
                             algotext, analyze, before_trading_start,
                             benchmark_returns, benchmark_sid, blotter,
                             bundle_data, capital_base, data, data_frequency,
                             emission_rate, handle_data, initialize,
                             metrics_set, namespace, trading_calendar)

    perf = algo.run()

    if output == '-':
        click.echo(str(perf))
    elif output != os.devnull:  # make the zipline magic not write any data
        perf.to_pickle(output)

    return perf
Example #35
0
    def init_instance_fixtures(self):
        super(TestDataPortal, self).init_instance_fixtures()

        self.data_portal = DataPortal(self.env)