def init_class_fixtures(cls):
        super(SecurityListTestCase, cls).init_class_fixtures()
        # this is ugly, but we need to create two different
        # TradingEnvironment/DataPortal pairs

        cls.start = pd.Timestamp(list(LEVERAGED_ETFS.keys())[0])
        end = pd.Timestamp('2015-02-17', tz='utc')
        cls.extra_knowledge_date = pd.Timestamp('2015-01-27', tz='utc')
        cls.trading_day_before_first_kd = pd.Timestamp('2015-01-23', tz='utc')
        symbols = ['AAPL', 'GOOG', 'BZQ', 'URTY', 'JFT']

        cls.env = cls.enter_class_context(tmp_trading_env(
            equities=pd.DataFrame.from_records([{
                'start_date': cls.start,
                'end_date': end,
                'symbol': symbol,
                'exchange': "TEST",
            } for symbol in symbols]),
            load=cls.make_load_function(),
        ))

        cls.sim_params = factory.create_simulation_parameters(
            start=cls.start,
            num_days=4,
            trading_calendar=cls.trading_calendar
        )

        cls.sim_params2 = sp2 = factory.create_simulation_parameters(
            start=cls.trading_day_before_first_kd, num_days=4
        )

        cls.env2 = cls.enter_class_context(tmp_trading_env(
            equities=pd.DataFrame.from_records([{
                'start_date': sp2.start_session,
                'end_date': sp2.end_session,
                'symbol': symbol,
                'exchange': "TEST",
            } for symbol in symbols]),
            load=cls.make_load_function(),
        ))

        cls.tempdir = cls.enter_class_context(tmp_dir())
        cls.tempdir2 = cls.enter_class_context(tmp_dir())

        cls.data_portal = create_data_portal(
            asset_finder=cls.env.asset_finder,
            tempdir=cls.tempdir,
            sim_params=cls.sim_params,
            sids=range(0, 5),
            trading_calendar=cls.trading_calendar,
        )

        cls.data_portal2 = create_data_portal(
            asset_finder=cls.env2.asset_finder,
            tempdir=cls.tempdir2,
            sim_params=cls.sim_params2,
            sids=range(0, 5),
            trading_calendar=cls.trading_calendar,
        )
Beispiel #2
0
    def init_class_fixtures(cls):
        super(SecurityListTestCase, cls).init_class_fixtures()
        # this is ugly, but we need to create two different
        # TradingEnvironment/DataPortal pairs

        cls.start = pd.Timestamp(list(LEVERAGED_ETFS.keys())[0])
        end = pd.Timestamp('2015-02-17', tz='utc')
        cls.extra_knowledge_date = pd.Timestamp('2015-01-27', tz='utc')
        cls.trading_day_before_first_kd = pd.Timestamp('2015-01-23', tz='utc')
        symbols = ['AAPL', 'GOOG', 'BZQ', 'URTY', 'JFT']

        cls.env = cls.enter_class_context(
            tmp_trading_env(equities=pd.DataFrame.from_records([{
                'start_date':
                cls.start,
                'end_date':
                end,
                'symbol':
                symbol,
                'exchange':
                "TEST",
            } for symbol in symbols]), ))

        cls.sim_params = factory.create_simulation_parameters(
            start=cls.start, num_days=4, trading_calendar=cls.trading_calendar)

        cls.sim_params2 = sp2 = factory.create_simulation_parameters(
            start=cls.trading_day_before_first_kd, num_days=4)

        cls.env2 = cls.enter_class_context(
            tmp_trading_env(equities=pd.DataFrame.from_records([{
                'start_date':
                sp2.start_session,
                'end_date':
                sp2.end_session,
                'symbol':
                symbol,
                'exchange':
                "TEST",
            } for symbol in symbols]), ))

        cls.tempdir = cls.enter_class_context(tmp_dir())
        cls.tempdir2 = cls.enter_class_context(tmp_dir())

        cls.data_portal = create_data_portal(
            asset_finder=cls.env.asset_finder,
            tempdir=cls.tempdir,
            sim_params=cls.sim_params,
            sids=range(0, 5),
            trading_calendar=cls.trading_calendar,
        )

        cls.data_portal2 = create_data_portal(
            asset_finder=cls.env2.asset_finder,
            tempdir=cls.tempdir2,
            sim_params=cls.sim_params2,
            sids=range(0, 5),
            trading_calendar=cls.trading_calendar,
        )
    def test_algo_without_rl_violation_after_delete(self):
        sim_params = factory.create_simulation_parameters(
            start=self.extra_knowledge_date,
            num_days=4,
        )
        equities = pd.DataFrame.from_records([{
            'symbol': 'BZQ',
            'start_date': sim_params.start_session,
            'end_date': sim_params.end_session,
            'exchange': "TEST",
        }])
        with TempDirectory() as new_tempdir, \
                security_list_copy(), \
                tmp_trading_env(equities=equities) as env:
            # add a delete statement removing bzq
            # write a new delete statement file to disk
            add_security_data([], ['BZQ'])

            data_portal = create_data_portal(
                env.asset_finder,
                new_tempdir,
                sim_params,
                range(0, 5),
                trading_calendar=self.trading_calendar,
            )

            algo = RestrictedAlgoWithoutCheck(
                symbol='BZQ', sim_params=sim_params, env=env
            )
            algo.run(data_portal)
Beispiel #4
0
    def test_algo_without_rl_violation_after_delete(self):
        sim_params = factory.create_simulation_parameters(
            start=self.extra_knowledge_date,
            num_days=4,
        )
        equities = pd.DataFrame.from_records([{
            'symbol':
            'BZQ',
            'start_date':
            sim_params.period_start,
            'end_date':
            sim_params.period_end,
        }])
        with TempDirectory() as new_tempdir, \
                security_list_copy(), \
                tmp_trading_env(equities=equities) as env:
            # add a delete statement removing bzq
            # write a new delete statement file to disk
            add_security_data([], ['BZQ'])

            data_portal = create_data_portal(
                env.asset_finder,
                new_tempdir,
                sim_params,
                range(0, 5),
                trading_schedule=self.trading_schedule,
            )

            algo = RestrictedAlgoWithoutCheck(symbol='BZQ',
                                              sim_params=sim_params,
                                              env=env)
            algo.run(data_portal)
Beispiel #5
0
    def transaction_sim(self, **params):
        """This is a utility method that asserts expected
        results for conversion of orders to transactions given a
        trade history
        """
        trade_count = params['trade_count']
        trade_interval = params['trade_interval']
        order_count = params['order_count']
        order_amount = params['order_amount']
        order_interval = params['order_interval']
        expected_txn_count = params['expected_txn_count']
        expected_txn_volume = params['expected_txn_volume']

        # optional parameters
        # ---------------------
        # if present, alternate between long and short sales
        alternate = params.get('alternate')

        # if present, expect transaction amounts to match orders exactly.
        complete_fill = params.get('complete_fill')

        sid = 1
        metadata = make_simple_equity_info([sid], self.start, self.end)
        with TempDirectory() as tempdir, \
                tmp_trading_env(equities=metadata) as env:

            if trade_interval < timedelta(days=1):
                sim_params = factory.create_simulation_parameters(
                    start=self.start, end=self.end, data_frequency="minute")

                minutes = env.market_minute_window(
                    sim_params.first_open,
                    int((trade_interval.total_seconds() / 60) * trade_count) +
                    100)

                price_data = np.array([10.1] * len(minutes))
                assets = {
                    sid:
                    pd.DataFrame({
                        "open": price_data,
                        "high": price_data,
                        "low": price_data,
                        "close": price_data,
                        "volume": np.array([100] * len(minutes)),
                        "dt": minutes
                    }).set_index("dt")
                }

                write_bcolz_minute_data(
                    env, env.days_in_range(minutes[0], minutes[-1]),
                    tempdir.path, assets)

                equity_minute_reader = BcolzMinuteBarReader(tempdir.path)

                data_portal = DataPortal(
                    env,
                    equity_minute_reader=equity_minute_reader,
                )
            else:
                sim_params = factory.create_simulation_parameters(
                    data_frequency="daily")

                days = sim_params.trading_days

                assets = {
                    1:
                    pd.DataFrame(
                        {
                            "open": [10.1] * len(days),
                            "high": [10.1] * len(days),
                            "low": [10.1] * len(days),
                            "close": [10.1] * len(days),
                            "volume": [100] * len(days),
                            "day": [day.value for day in days]
                        },
                        index=days)
                }

                path = os.path.join(tempdir.path, "testdata.bcolz")
                BcolzDailyBarWriter(path, days).write(assets.items())

                equity_daily_reader = BcolzDailyBarReader(path)

                data_portal = DataPortal(
                    env,
                    equity_daily_reader=equity_daily_reader,
                )

            if "default_slippage" not in params or \
               not params["default_slippage"]:
                slippage_func = FixedSlippage()
            else:
                slippage_func = None

            blotter = Blotter(sim_params.data_frequency, self.env.asset_finder,
                              slippage_func)

            start_date = sim_params.first_open

            if alternate:
                alternator = -1
            else:
                alternator = 1

            tracker = PerformanceTracker(sim_params, self.env)

            # replicate what tradesim does by going through every minute or day
            # of the simulation and processing open orders each time
            if sim_params.data_frequency == "minute":
                ticks = minutes
            else:
                ticks = days

            transactions = []

            order_list = []
            order_date = start_date
            for tick in ticks:
                blotter.current_dt = tick
                if tick >= order_date and len(order_list) < order_count:
                    # place an order
                    direction = alternator**len(order_list)
                    order_id = blotter.order(
                        blotter.asset_finder.retrieve_asset(sid),
                        order_amount * direction, MarketOrder())
                    order_list.append(blotter.orders[order_id])
                    order_date = order_date + order_interval
                    # move after market orders to just after market next
                    # market open.
                    if order_date.hour >= 21:
                        if order_date.minute >= 00:
                            order_date = order_date + timedelta(days=1)
                            order_date = order_date.replace(hour=14, minute=30)
                else:
                    bar_data = BarData(data_portal, lambda: tick,
                                       sim_params.data_frequency)
                    txns, _ = blotter.get_transactions(bar_data)
                    for txn in txns:
                        tracker.process_transaction(txn)
                        transactions.append(txn)

            for i in range(order_count):
                order = order_list[i]
                self.assertEqual(order.sid, sid)
                self.assertEqual(order.amount, order_amount * alternator**i)

            if complete_fill:
                self.assertEqual(len(transactions), len(order_list))

            total_volume = 0
            for i in range(len(transactions)):
                txn = transactions[i]
                total_volume += txn.amount
                if complete_fill:
                    order = order_list[i]
                    self.assertEqual(order.amount, txn.amount)

            self.assertEqual(total_volume, expected_txn_volume)

            self.assertEqual(len(transactions), expected_txn_count)

            cumulative_pos = tracker.position_tracker.positions[sid]
            if total_volume == 0:
                self.assertIsNone(cumulative_pos)
            else:
                self.assertEqual(total_volume, cumulative_pos.amount)

            # the open orders should not contain sid.
            oo = blotter.open_orders
            self.assertNotIn(sid, oo, "Entry is removed when no open orders")
Beispiel #6
0
    def transaction_sim(self, **params):
        """This is a utility method that asserts expected
        results for conversion of orders to transactions given a
        trade history
        """
        trade_count = params["trade_count"]
        trade_interval = params["trade_interval"]
        order_count = params["order_count"]
        order_amount = params["order_amount"]
        order_interval = params["order_interval"]
        expected_txn_count = params["expected_txn_count"]
        expected_txn_volume = params["expected_txn_volume"]

        # optional parameters
        # ---------------------
        # if present, alternate between long and short sales
        alternate = params.get("alternate")

        # if present, expect transaction amounts to match orders exactly.
        complete_fill = params.get("complete_fill")

        sid = 1
        metadata = make_simple_equity_info([sid], self.start, self.end)
        with TempDirectory() as tempdir, tmp_trading_env(equities=metadata) as env:

            if trade_interval < timedelta(days=1):
                sim_params = factory.create_simulation_parameters(
                    start=self.start, end=self.end, data_frequency="minute"
                )

                minutes = env.market_minute_window(
                    sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100
                )

                price_data = np.array([10.1] * len(minutes))
                assets = {
                    sid: pd.DataFrame(
                        {
                            "open": price_data,
                            "high": price_data,
                            "low": price_data,
                            "close": price_data,
                            "volume": np.array([100] * len(minutes)),
                            "dt": minutes,
                        }
                    ).set_index("dt")
                }

                write_bcolz_minute_data(
                    env, env.days_in_range(minutes[0], minutes[-1]), tempdir.path, iteritems(assets)
                )

                equity_minute_reader = BcolzMinuteBarReader(tempdir.path)

                data_portal = DataPortal(
                    env,
                    first_trading_day=equity_minute_reader.first_trading_day,
                    equity_minute_reader=equity_minute_reader,
                )
            else:
                sim_params = factory.create_simulation_parameters(data_frequency="daily")

                days = sim_params.trading_days

                assets = {
                    1: pd.DataFrame(
                        {
                            "open": [10.1] * len(days),
                            "high": [10.1] * len(days),
                            "low": [10.1] * len(days),
                            "close": [10.1] * len(days),
                            "volume": [100] * len(days),
                            "day": [day.value for day in days],
                        },
                        index=days,
                    )
                }

                path = os.path.join(tempdir.path, "testdata.bcolz")
                BcolzDailyBarWriter(path, days).write(assets.items())

                equity_daily_reader = BcolzDailyBarReader(path)

                data_portal = DataPortal(
                    env,
                    first_trading_day=equity_daily_reader.first_trading_day,
                    equity_daily_reader=equity_daily_reader,
                )

            if "default_slippage" not in params or not params["default_slippage"]:
                slippage_func = FixedSlippage()
            else:
                slippage_func = None

            blotter = Blotter(sim_params.data_frequency, self.env.asset_finder, slippage_func)

            start_date = sim_params.first_open

            if alternate:
                alternator = -1
            else:
                alternator = 1

            tracker = PerformanceTracker(sim_params, self.env)

            # replicate what tradesim does by going through every minute or day
            # of the simulation and processing open orders each time
            if sim_params.data_frequency == "minute":
                ticks = minutes
            else:
                ticks = days

            transactions = []

            order_list = []
            order_date = start_date
            for tick in ticks:
                blotter.current_dt = tick
                if tick >= order_date and len(order_list) < order_count:
                    # place an order
                    direction = alternator ** len(order_list)
                    order_id = blotter.order(
                        blotter.asset_finder.retrieve_asset(sid), order_amount * direction, MarketOrder()
                    )
                    order_list.append(blotter.orders[order_id])
                    order_date = order_date + order_interval
                    # move after market orders to just after market next
                    # market open.
                    if order_date.hour >= 21:
                        if order_date.minute >= 00:
                            order_date = order_date + timedelta(days=1)
                            order_date = order_date.replace(hour=14, minute=30)
                else:
                    bar_data = BarData(data_portal, lambda: tick, sim_params.data_frequency)
                    txns, _, closed_orders = blotter.get_transactions(bar_data)
                    for txn in txns:
                        tracker.process_transaction(txn)
                        transactions.append(txn)

                    blotter.prune_orders(closed_orders)

            for i in range(order_count):
                order = order_list[i]
                self.assertEqual(order.sid, sid)
                self.assertEqual(order.amount, order_amount * alternator ** i)

            if complete_fill:
                self.assertEqual(len(transactions), len(order_list))

            total_volume = 0
            for i in range(len(transactions)):
                txn = transactions[i]
                total_volume += txn.amount
                if complete_fill:
                    order = order_list[i]
                    self.assertEqual(order.amount, txn.amount)

            self.assertEqual(total_volume, expected_txn_volume)

            self.assertEqual(len(transactions), expected_txn_count)

            cumulative_pos = tracker.position_tracker.positions[sid]
            if total_volume == 0:
                self.assertIsNone(cumulative_pos)
            else:
                self.assertEqual(total_volume, cumulative_pos.amount)

            # the open orders should not contain sid.
            oo = blotter.open_orders
            self.assertNotIn(sid, oo, "Entry is removed when no open orders")
Beispiel #7
0
    def test_ingest(self):
        zipline_root = self.enter_instance_context(tmp_dir()).path
        env = self.enter_instance_context(tmp_trading_env())

        start = pd.Timestamp('2014-01-06', tz='utc')
        end = pd.Timestamp('2014-01-10', tz='utc')
        calendar = trading_days[trading_days.slice_indexer(start, end)]
        minutes = env.minutes_for_days_in_range(calendar[0], calendar[-1])
        outer_environ = {
            'ZIPLINE_ROOT': zipline_root,
        }

        sids = tuple(range(3))
        equities = make_simple_equity_info(
            sids,
            calendar[0],
            calendar[-1],
        )

        daily_bar_data = make_bar_data(equities, calendar)
        minute_bar_data = make_bar_data(equities, minutes)
        first_split_ratio = 0.5
        second_split_ratio = 0.1
        splits = pd.DataFrame.from_records([
            {
                'effective_date': str_to_seconds('2014-01-08'),
                'ratio': first_split_ratio,
                'sid': 0,
            },
            {
                'effective_date': str_to_seconds('2014-01-09'),
                'ratio': second_split_ratio,
                'sid': 1,
            },
        ])

        @self.register('bundle',
                       calendar=calendar,
                       opens=env.opens_in_range(calendar[0], calendar[-1]),
                       closes=env.closes_in_range(calendar[0], calendar[-1]))
        def bundle_ingest(environ,
                          asset_db_writer,
                          minute_bar_writer,
                          daily_bar_writer,
                          adjustment_writer,
                          calendar,
                          cache,
                          show_progress):
            assert_is(environ, outer_environ)

            asset_db_writer.write(equities=equities)
            minute_bar_writer.write(minute_bar_data)
            daily_bar_writer.write(daily_bar_data)
            adjustment_writer.write(splits=splits)

            assert_is_instance(calendar, pd.DatetimeIndex)
            assert_is_instance(cache, dataframe_cache)
            assert_is_instance(show_progress, bool)

        self.ingest('bundle', environ=outer_environ)
        bundle = load('bundle', environ=outer_environ)

        assert_equal(set(bundle.asset_finder.sids), set(sids))

        columns = 'open', 'high', 'low', 'close', 'volume'

        actual = bundle.minute_bar_reader.load_raw_arrays(
            columns,
            minutes[0],
            minutes[-1],
            sids,
        )

        for actual_column, colname in zip(actual, columns):
            assert_equal(
                actual_column,
                expected_bar_values_2d(minutes, equities, colname),
                msg=colname,
            )

        actual = bundle.daily_bar_reader.load_raw_arrays(
            columns,
            calendar[0],
            calendar[-1],
            sids,
        )
        for actual_column, colname in zip(actual, columns):
            assert_equal(
                actual_column,
                expected_bar_values_2d(calendar, equities, colname),
                msg=colname,
            )
        adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
            columns,
            calendar,
            pd.Index(sids),
        )
        for column, adjustments in zip(columns, adjustments_for_cols[:-1]):
            # iterate over all the adjustments but `volume`
            assert_equal(
                adjustments,
                {
                    2: [Float64Multiply(
                        first_row=0,
                        last_row=2,
                        first_col=0,
                        last_col=0,
                        value=first_split_ratio,
                    )],
                    3: [Float64Multiply(
                        first_row=0,
                        last_row=3,
                        first_col=1,
                        last_col=1,
                        value=second_split_ratio,
                    )],
                },
                msg=column,
            )

        # check the volume, the value should be 1/ratio
        assert_equal(
            adjustments_for_cols[-1],
            {
                2: [Float64Multiply(
                    first_row=0,
                    last_row=2,
                    first_col=0,
                    last_col=0,
                    value=1 / first_split_ratio,
                )],
                3: [Float64Multiply(
                    first_row=0,
                    last_row=3,
                    first_col=1,
                    last_col=1,
                    value=1 / second_split_ratio,
                )],
            },
            msg='volume',
        )
Beispiel #8
0
    def test_ingest(self):
        env = self.enter_instance_context(tmp_trading_env())

        start = pd.Timestamp('2014-01-06', tz='utc')
        end = pd.Timestamp('2014-01-10', tz='utc')
        calendar = trading_days[trading_days.slice_indexer(start, end)]
        minutes = env.minutes_for_days_in_range(calendar[0], calendar[-1])

        sids = tuple(range(3))
        equities = make_simple_equity_info(
            sids,
            calendar[0],
            calendar[-1],
        )

        daily_bar_data = make_bar_data(equities, calendar)
        minute_bar_data = make_bar_data(equities, minutes)
        first_split_ratio = 0.5
        second_split_ratio = 0.1
        splits = pd.DataFrame.from_records([
            {
                'effective_date': str_to_seconds('2014-01-08'),
                'ratio': first_split_ratio,
                'sid': 0,
            },
            {
                'effective_date': str_to_seconds('2014-01-09'),
                'ratio': second_split_ratio,
                'sid': 1,
            },
        ])

        @self.register('bundle',
                       calendar=calendar,
                       opens=env.opens_in_range(calendar[0], calendar[-1]),
                       closes=env.closes_in_range(calendar[0], calendar[-1]))
        def bundle_ingest(environ,
                          asset_db_writer,
                          minute_bar_writer,
                          daily_bar_writer,
                          adjustment_writer,
                          calendar,
                          cache,
                          show_progress,
                          output_dir):
            assert_is(environ, self.environ)

            asset_db_writer.write(equities=equities)
            minute_bar_writer.write(minute_bar_data)
            daily_bar_writer.write(daily_bar_data)
            adjustment_writer.write(splits=splits)

            assert_is_instance(calendar, pd.DatetimeIndex)
            assert_is_instance(cache, dataframe_cache)
            assert_is_instance(show_progress, bool)

        self.ingest('bundle', environ=self.environ)
        bundle = self.load('bundle', environ=self.environ)

        assert_equal(set(bundle.asset_finder.sids), set(sids))

        columns = 'open', 'high', 'low', 'close', 'volume'

        actual = bundle.minute_bar_reader.load_raw_arrays(
            columns,
            minutes[0],
            minutes[-1],
            sids,
        )

        for actual_column, colname in zip(actual, columns):
            assert_equal(
                actual_column,
                expected_bar_values_2d(minutes, equities, colname),
                msg=colname,
            )

        actual = bundle.daily_bar_reader.load_raw_arrays(
            columns,
            calendar[0],
            calendar[-1],
            sids,
        )
        for actual_column, colname in zip(actual, columns):
            assert_equal(
                actual_column,
                expected_bar_values_2d(calendar, equities, colname),
                msg=colname,
            )
        adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
            columns,
            calendar,
            pd.Index(sids),
        )
        for column, adjustments in zip(columns, adjustments_for_cols[:-1]):
            # iterate over all the adjustments but `volume`
            assert_equal(
                adjustments,
                {
                    2: [Float64Multiply(
                        first_row=0,
                        last_row=2,
                        first_col=0,
                        last_col=0,
                        value=first_split_ratio,
                    )],
                    3: [Float64Multiply(
                        first_row=0,
                        last_row=3,
                        first_col=1,
                        last_col=1,
                        value=second_split_ratio,
                    )],
                },
                msg=column,
            )

        # check the volume, the value should be 1/ratio
        assert_equal(
            adjustments_for_cols[-1],
            {
                2: [Float64Multiply(
                    first_row=0,
                    last_row=2,
                    first_col=0,
                    last_col=0,
                    value=1 / first_split_ratio,
                )],
                3: [Float64Multiply(
                    first_row=0,
                    last_row=3,
                    first_col=1,
                    last_col=1,
                    value=1 / second_split_ratio,
                )],
            },
            msg='volume',
        )