def func(case):
            return case[0].__name__

    for case in cases:
        new_case = list(case)
        key = func(case)
        new_case.insert(0, key)
        results.append(new_case)
    return results


sim_params_daily = SimulationParameters(datetime.datetime(2013,
                                                          6,
                                                          19,
                                                          tzinfo=pytz.UTC),
                                        datetime.datetime(2013,
                                                          6,
                                                          19,
                                                          tzinfo=pytz.UTC),
                                        10000,
                                        emission_rate='daily')
sim_params_minute = SimulationParameters(datetime.datetime(2013,
                                                           6,
                                                           19,
                                                           tzinfo=pytz.UTC),
                                         datetime.datetime(2013,
                                                           6,
                                                           19,
                                                           tzinfo=pytz.UTC),
                                         10000,
                                         emission_rate='minute')
returns = factory.create_returns_from_list([1.0], sim_params_daily)
Example #2
0
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
         algotext, defines, data_frequency, capital_base, bundle,
         bundle_timestamp, start, end, output, trading_calendar, print_algo,
         metrics_set, local_namespace, environ, blotter, benchmark_returns,
         broker, state_filename, realtime_bar_target, performance_callback,
         stop_execution_callback, teardown, execution_id):
    """
    Run a backtest for the given algorithm.
    This is shared between the cli and :func:`zipline.run_algo`.

    zipline-live additions:
    broker - wrapper to connect to a real broker
    state_filename - saving the context of the algo to be able to restart
    performance_callback - a callback to send performance results everyday and not only at the end of the backtest.
        this allows to run live, and monitor the performance of the algorithm
    stop_execution_callback - A callback to check if execution should be stopped. it is used to be able to stop live
        trading (also simulation could be stopped using this) execution. if the callback returns True, then algo
        execution will be aborted.
    teardown - algo method like handle_data() or before_trading_start() that is called when the algo execution stops
    execution_id - unique id to identify this execution (backtest or live instance)
    """
    if benchmark_returns is None:
        benchmark_returns, _ = load_market_data(environ=environ)

    emission_rate = 'daily'
    if broker:
        emission_rate = 'minute'
        # if we run zipline as a command line tool, these will probably not be initiated
        if not start:
            start = pd.Timestamp.utcnow()
        if not end:
            # in cli mode, sessions are 1 day only. and it will be re-ran each day by user
            end = start + pd.Timedelta('1 day')

    if algotext is not None:
        if local_namespace:
            ip = get_ipython()  # noqa
            namespace = ip.user_ns
        else:
            namespace = {}

        for assign in defines:
            try:
                name, value = assign.split('=', 2)
            except ValueError:
                raise ValueError(
                    'invalid define %r, should be of the form name=value' %
                    assign, )
            try:
                # evaluate in the same namespace so names may refer to
                # eachother
                namespace[name] = eval(value, namespace)
            except Exception as e:
                raise ValueError(
                    'failed to execute definition for name %r: %s' %
                    (name, e), )
    elif defines:
        raise _RunAlgoError(
            'cannot pass define without `algotext`',
            "cannot pass '-D' / '--define' without '-t' / '--algotext'",
        )
    else:
        namespace = {}
        if algofile is not None:
            algotext = algofile.read()

    if print_algo:
        if PYGMENTS:
            highlight(
                algotext,
                PythonLexer(),
                TerminalFormatter(),
                outfile=sys.stdout,
            )
        else:
            click.echo(algotext)

    if trading_calendar is None:
        trading_calendar = get_calendar('NYSE')

    # date parameter validation
    if trading_calendar.session_distance(start, end) < 1:
        raise _RunAlgoError(
            'There are no trading days between %s and %s' % (
                start.date(),
                end.date(),
            ), )

    bundle_data = bundles.load(
        bundle,
        environ,
        bundle_timestamp,
    )

    first_trading_day = \
        bundle_data.equity_minute_bar_reader.first_trading_day

    DataPortalClass = (partial(DataPortalLive, broker)
                       if broker else DataPortal)

    data = DataPortalClass(
        bundle_data.asset_finder,
        trading_calendar=trading_calendar,
        first_trading_day=first_trading_day,
        equity_minute_reader=bundle_data.equity_minute_bar_reader,
        equity_daily_reader=bundle_data.equity_daily_bar_reader,
        adjustment_reader=bundle_data.adjustment_reader,
    )

    pipeline_loader = USEquityPricingLoader(
        bundle_data.equity_daily_bar_reader,
        bundle_data.adjustment_reader,
    )

    def choose_loader(column):
        if column in USEquityPricing.columns:
            return pipeline_loader
        raise ValueError("No PipelineLoader registered for column %s." %
                         column)

    if isinstance(metrics_set, six.string_types):
        try:
            metrics_set = metrics.load(metrics_set)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    if isinstance(blotter, six.string_types):
        try:
            blotter = load(Blotter, blotter)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    TradingAlgorithmClass = (partial(LiveTradingAlgorithm,
                                     broker=broker,
                                     state_filename=state_filename,
                                     realtime_bar_target=realtime_bar_target)
                             if broker else TradingAlgorithm)

    perf = TradingAlgorithmClass(
        namespace=namespace,
        data_portal=data,
        get_pipeline_loader=choose_loader,
        trading_calendar=trading_calendar,
        sim_params=SimulationParameters(start_session=start,
                                        end_session=end,
                                        trading_calendar=trading_calendar,
                                        capital_base=capital_base,
                                        emission_rate=emission_rate,
                                        data_frequency=data_frequency,
                                        execution_id=execution_id),
        metrics_set=metrics_set,
        blotter=blotter,
        benchmark_returns=benchmark_returns,
        performance_callback=performance_callback,
        stop_execution_callback=stop_execution_callback,
        **{
            'initialize': initialize,
            'handle_data': handle_data,
            'before_trading_start': before_trading_start,
            'analyze': analyze,
            'teardown': teardown,
        } if algotext is None else {
            'algo_filename': getattr(algofile, 'name', '<algorithm>'),
            'script': algotext,
        }).run()

    if output == '-':
        click.echo(str(perf))
    elif output != os.devnull:  # make the zipline magic not write any data
        perf.to_pickle(output)

    return perf
Example #3
0
 def setUp(self):
     self.dt = datetime.datetime(2003, 11, 30, tzinfo=pytz.utc)
     self.end_dt = datetime.datetime(2004, 11, 25, tzinfo=pytz.utc)
     self.sim_params = SimulationParameters(self.dt, self.end_dt)
     self.benchmark_events = benchmark_events_in_range(self.sim_params)
Example #4
0
    plt.legend(loc=0)
    plt.show()


# Load data manually
start = datetime(2000, 1, 1, 0, 0, 0, 0, pytz.utc).date()
end = datetime(2017, 7, 1, 0, 0, 0, 0, pytz.utc).date()
data = load(['SPY'], start=start, end=end)
"""
Prepare simulation
"""
sim_params = SimulationParameters(
    start_session=pd.Timestamp('2004-01-01', tz=pytz.utc),
    end_session=pd.Timestamp('2017-07-01', tz=pytz.utc),
    capital_base=1.0e5,
    data_frequency='daily',
    trading_calendar=get_calendar("NYSE"),
)

period = [3]
bottom = [30]
top = [70]
shift = [15]
max_long = [5]
prob_window = [10]
prob = [20]


def worker(args):
    """thread worker function"""
    def test_handle_adjustment(self, set_screen):
        AAPL, MSFT, BRK_A = assets = self.assets

        window_lengths = [1, 2, 5, 10]
        vwaps = self.compute_expected_vwaps(window_lengths)

        def vwap_key(length):
            return "vwap_%d" % length

        def initialize(context):
            pipeline = Pipeline()
            context.vwaps = []
            for length in vwaps:
                name = vwap_key(length)
                factor = VWAP(window_length=length)
                context.vwaps.append(factor)
                pipeline.add(factor, name=name)

            filter_ = (USEquityPricing.close.latest > 300)
            pipeline.add(filter_, 'filter')
            if set_screen:
                pipeline.set_screen(filter_)

            attach_pipeline(pipeline, 'test')

        def handle_data(context, data):
            today = normalize_date(get_datetime())
            results = pipeline_output('test')
            expect_over_300 = {
                AAPL: today < self.AAPL_split_date,
                MSFT: False,
                BRK_A: True,
            }
            for asset in assets:
                should_pass_filter = expect_over_300[asset]
                if set_screen and not should_pass_filter:
                    self.assertNotIn(asset, results.index)
                    continue

                asset_results = results.loc[asset]
                self.assertEqual(asset_results['filter'], should_pass_filter)
                for length in vwaps:
                    computed = results.loc[asset, vwap_key(length)]
                    expected = vwaps[length][asset].loc[today]
                    # Only having two places of precision here is a bit
                    # unfortunate.
                    assert_almost_equal(computed, expected, decimal=2)

        # Do the same checks in before_trading_start
        before_trading_start = handle_data

        self.run_algorithm(
            initialize=initialize,
            handle_data=handle_data,
            before_trading_start=before_trading_start,
            sim_params=SimulationParameters(
                start_session=self.dates[max(window_lengths)],
                end_session=self.dates[-1],
                data_frequency='daily',
                emission_rate='daily',
                trading_calendar=self.trading_calendar,
            )
        )
Example #6
0
 def test_multi_source_as_input(self):
     sim_params = SimulationParameters(self.df.index[0], self.df.index[-1])
     algo = TestRegisterTransformAlgorithm(sim_params=sim_params,
                                           sids=[0, 1, 133])
     algo.run([self.source, self.df_source], overwrite_sim_params=False)
     self.assertEqual(len(algo.sources), 2)
Example #7
0
    def test_tracker(self, parameter_comment, days_to_delete):
        """
        @days_to_delete - configures which days in the data set we should
        remove, used for ensuring that we still return performance messages
        even when there is no data.
        """
        # This date range covers Columbus day,
        # however Columbus day is not a market holiday
        #
        #     October 2008
        # Su Mo Tu We Th Fr Sa
        #           1  2  3  4
        #  5  6  7  8  9 10 11
        # 12 13 14 15 16 17 18
        # 19 20 21 22 23 24 25
        # 26 27 28 29 30 31
        start_dt = datetime.datetime(year=2008,
                                     month=10,
                                     day=9,
                                     tzinfo=pytz.utc)
        end_dt = datetime.datetime(year=2008,
                                   month=10,
                                   day=16,
                                   tzinfo=pytz.utc)

        trade_count = 6
        sid = 133
        price = 10.1
        price_list = [price] * trade_count
        volume = [100] * trade_count
        trade_time_increment = datetime.timedelta(days=1)

        sim_params = SimulationParameters(
            period_start=start_dt,
            period_end=end_dt
        )

        trade_history = factory.create_trade_history(
            sid,
            price_list,
            volume,
            trade_time_increment,
            sim_params,
            source_id="factory1"
        )

        sid2 = 134
        price2 = 12.12
        price2_list = [price2] * trade_count
        trade_history2 = factory.create_trade_history(
            sid2,
            price2_list,
            volume,
            trade_time_increment,
            sim_params,
            source_id="factory2"
        )
        # 'middle' start of 3 depends on number of days == 7
        middle = 3

        # First delete from middle
        if days_to_delete.middle:
            del trade_history[middle:(middle + days_to_delete.middle)]
            del trade_history2[middle:(middle + days_to_delete.middle)]

        # Delete start
        if days_to_delete.start:
            del trade_history[:days_to_delete.start]
            del trade_history2[:days_to_delete.start]

        # Delete from end
        if days_to_delete.end:
            del trade_history[-days_to_delete.end:]
            del trade_history2[-days_to_delete.end:]

        sim_params.first_open = \
            sim_params.calculate_first_open()
        sim_params.last_close = \
            sim_params.calculate_last_close()
        sim_params.capital_base = 1000.0
        sim_params.frame_index = [
            'sid',
            'volume',
            'dt',
            'price',
            'changed']
        perf_tracker = perf.PerformanceTracker(
            sim_params
        )

        events = date_sorted_sources(trade_history, trade_history2)

        events = [self.event_with_txn(event, trade_history[0].dt)
                  for event in events]

        # Extract events with transactions to use for verification.
        events_with_txns = [event for event in events if event.TRANSACTION]

        perf_messages = \
            [msg for date, snapshot in
             perf_tracker.transform(
                 itertools.groupby(events, attrgetter('dt')))
             for event in snapshot
             for msg in event.perf_messages]

        end_perf_messages, risk_message = perf_tracker.handle_simulation_end()

        perf_messages.extend(end_perf_messages)

        #we skip two trades, to test case of None transaction
        self.assertEqual(perf_tracker.txn_count, len(events_with_txns))

        cumulative_pos = perf_tracker.cumulative_performance.positions[sid]
        expected_size = len(events_with_txns) / 2 * -25
        self.assertEqual(cumulative_pos.amount, expected_size)

        self.assertEqual(perf_tracker.last_close,
                         perf_tracker.cumulative_risk_metrics.end_date)

        self.assertEqual(len(perf_messages),
                         sim_params.days_in_period)
Example #8
0
    def test_minute_buy_and_hold(self):
        with trading.TradingEnvironment():
            start_date = datetime.datetime(
                year=2006,
                month=1,
                day=3,
                hour=0,
                minute=0,
                tzinfo=pytz.utc)
            end_date = datetime.datetime(
                year=2006,
                month=1,
                day=5,
                hour=0,
                minute=0,
                tzinfo=pytz.utc)

            sim_params = SimulationParameters(
                period_start=start_date,
                period_end=end_date,
                emission_rate='daily',
                data_frequency='minute')

            algo = BuyAndHoldAlgorithm(
                sim_params=sim_params,
                data_frequency='minute')

            first_date = datetime.datetime(2006, 1, 3, tzinfo=pytz.utc)
            first_open, first_close = \
                trading.environment.get_open_and_close(first_date)

            second_date = datetime.datetime(2006, 1, 4, tzinfo=pytz.utc)
            second_open, second_close = \
                trading.environment.get_open_and_close(second_date)

            third_date = datetime.datetime(2006, 1, 5, tzinfo=pytz.utc)
            third_open, third_close = \
                trading.environment.get_open_and_close(third_date)

            benchmark_data = [
                Event({
                    'returns': 0.1,
                    'dt': first_close,
                    'source_id': 'test-benchmark-source',
                    'type': DATASOURCE_TYPE.BENCHMARK
                }),
                Event({
                    'returns': 0.2,
                    'dt': second_close,
                    'source_id': 'test-benchmark-source',
                    'type': DATASOURCE_TYPE.BENCHMARK
                }),
                Event({
                    'returns': 0.4,
                    'dt': third_close,
                    'source_id': 'test-benchmark-source',
                    'type': DATASOURCE_TYPE.BENCHMARK
                }),
            ]

            trade_bar_data = [
                Event({
                    'open_price': 10,
                    'close_price': 15,
                    'price': 15,
                    'volume': 1000,
                    'sid': 1,
                    'dt': first_open,
                    'source_id': 'test-trade-source',
                    'type': DATASOURCE_TYPE.TRADE
                }),
                Event({
                    'open_price': 10,
                    'close_price': 15,
                    'price': 15,
                    'volume': 1000,
                    'sid': 1,
                    'dt': first_open + datetime.timedelta(minutes=10),
                    'source_id': 'test-trade-source',
                    'type': DATASOURCE_TYPE.TRADE
                }),
                Event({
                    'open_price': 15,
                    'close_price': 20,
                    'price': 20,
                    'volume': 2000,
                    'sid': 1,
                    'dt': second_open,
                    'source_id': 'test-trade-source',
                    'type': DATASOURCE_TYPE.TRADE
                }),
                Event({
                    'open_price': 15,
                    'close_price': 20,
                    'price': 20,
                    'volume': 2000,
                    'sid': 1,
                    'dt': second_open + datetime.timedelta(minutes=10),
                    'source_id': 'test-trade-source',
                    'type': DATASOURCE_TYPE.TRADE
                }),
                Event({
                    'open_price': 20,
                    'close_price': 15,
                    'price': 15,
                    'volume': 1000,
                    'sid': 1,
                    'dt': third_open,
                    'source_id': 'test-trade-source',
                    'type': DATASOURCE_TYPE.TRADE
                }),
                Event({
                    'open_price': 20,
                    'close_price': 15,
                    'price': 15,
                    'volume': 1000,
                    'sid': 1,
                    'dt': third_open + datetime.timedelta(minutes=10),
                    'source_id': 'test-trade-source',
                    'type': DATASOURCE_TYPE.TRADE
                }),
            ]

            algo.benchmark_return_source = benchmark_data
            algo.sources = list([trade_bar_data])
            gen = algo._create_generator(sim_params)

            crm = algo.perf_tracker.cumulative_risk_metrics

            first_msg = gen.next()

            self.assertIsNotNone(first_msg,
                                 "There should be a message emitted.")

            # Protects against bug where the positions appeared to be
            # a day late, because benchmarks were triggering
            # calculations before the events for the day were
            # processed.
            self.assertEqual(1, len(algo.portfolio.positions), "There should "
                             "be one position after the first day.")

            self.assertTrue(
                np.isnan(crm.algorithm_volatility[-1]),
                "On the first day algorithm volatility does not exist.")

            second_msg = gen.next()

            self.assertIsNotNone(second_msg, "There should be a message "
                                 "emitted.")

            self.assertEqual(1, len(algo.portfolio.positions),
                             "Number of positions should stay the same.")

            # TODO: Hand derive. Current value is just a canary to
            # detect changes.
            np.testing.assert_almost_equal(
                0.050022510129558301,
                crm.algorithm_returns[-1],
                decimal=6)

            third_msg = gen.next()

            self.assertEqual(1, len(algo.portfolio.positions),
                             "Number of positions should stay the same.")

            self.assertIsNotNone(third_msg, "There should be a message "
                                 "emitted.")

            # TODO: Hand derive. Current value is just a canary to
            # detect changes.
            np.testing.assert_almost_equal(
                -0.047639464532418657,
                crm.algorithm_returns[-1],
                decimal=6)
Example #9
0
    def test_daily_buy_and_hold(self):

        start_date = datetime.datetime(
            year=2006,
            month=1,
            day=3,
            hour=0,
            minute=0,
            tzinfo=pytz.utc)
        end_date = datetime.datetime(
            year=2006,
            month=1,
            day=5,
            hour=0,
            minute=0,
            tzinfo=pytz.utc)

        sim_params = SimulationParameters(
            period_start=start_date,
            period_end=end_date,
            emission_rate='daily'
        )

        algo = BuyAndHoldAlgorithm(
            sim_params=sim_params,
            data_frequency='daily')

        first_date = datetime.datetime(2006, 1, 3, tzinfo=pytz.utc)
        second_date = datetime.datetime(2006, 1, 4, tzinfo=pytz.utc)
        third_date = datetime.datetime(2006, 1, 5, tzinfo=pytz.utc)

        trade_bar_data = [
            Event({
                'open_price': 10,
                'close_price': 15,
                'price': 15,
                'volume': 1000,
                'sid': 1,
                'dt': first_date,
                'source_id': 'test-trade-source',
                'type': DATASOURCE_TYPE.TRADE
            }),
            Event({
                'open_price': 15,
                'close_price': 20,
                'price': 20,
                'volume': 2000,
                'sid': 1,
                'dt': second_date,
                'source_id': 'test_list',
                'type': DATASOURCE_TYPE.TRADE
            }),
            Event({
                'open_price': 20,
                'close_price': 15,
                'price': 15,
                'volume': 1000,
                'sid': 1,
                'dt': third_date,
                'source_id': 'test_list',
                'type': DATASOURCE_TYPE.TRADE
            }),
        ]
        benchmark_data = [
            Event({
                'returns': 0.1,
                'dt': first_date,
                'source_id': 'test-benchmark-source',
                'type': DATASOURCE_TYPE.BENCHMARK
            }),
            Event({
                'returns': 0.2,
                'dt': second_date,
                'source_id': 'test-benchmark-source',
                'type': DATASOURCE_TYPE.BENCHMARK
            }),
            Event({
                'returns': 0.4,
                'dt': third_date,
                'source_id': 'test-benchmark-source',
                'type': DATASOURCE_TYPE.BENCHMARK
            }),
        ]

        algo.benchmark_return_source = benchmark_data
        algo.sources = list([trade_bar_data])
        gen = algo._create_generator(sim_params)

        # TODO: Hand derive these results.
        #       Currently, the output from the time of this writing to
        #       at least be an early warning against changes.
        expected_algorithm_returns = {
            first_date: 0.0,
            second_date: -0.000350,
            third_date: -0.050018
        }

        # TODO: Hand derive these results.
        #       Currently, the output from the time of this writing to
        #       at least be an early warning against changes.
        expected_sharpe = {
            first_date: np.nan,
            second_date: -1.630920,
            third_date: -1.016842,
        }

        for bar in gen:
            current_dt = algo.get_datetime()
            crm = algo.perf_tracker.cumulative_risk_metrics

            np.testing.assert_almost_equal(
                expected_algorithm_returns[current_dt],
                crm.algorithm_returns[-1],
                decimal=6)

            np.testing.assert_almost_equal(
                expected_sharpe[current_dt],
                crm.sharpe[-1],
                decimal=6)
Example #10
0
def run(opt='twse', debug=False, limit=0):
    maxlen = 30
    starttime = datetime.utcnow() - timedelta(days=300)
    endtime = datetime.utcnow()
    report = Report('bbands',
                    sort=[('buys', False), ('sells', False),
                          ('portfolio_value', False)],
                    limit=20)
    kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
    idhandler = TwseIdDBHandler(
        **kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
    for stockid in idhandler.stock.get_ids():
        try:
            kwargs = {
                'opt': opt,
                'targets': ['stock', 'future', 'credit'],
                'starttime': starttime,
                'endtime': endtime,
                'stockids': [stockid],
                'traderids': [],
                'base': 'stock',
                'callback': None,
                'limit': 1,
                'debug': debug
            }
            panel, dbhandler = collect_hisframe(**kwargs)
            if len(panel[stockid].index) < maxlen:
                continue

            sim_params = SimulationParameters(
                period_start=panel[stockid].index[0],
                period_end=panel[stockid].index[-1],
                data_frequency='daily',
                emission_rate='daily')

            bbands = BBands(dbhandler=dbhandler,
                            debug=debug,
                            sim_params=sim_params)
            results = bbands.run(panel).fillna(0)
            risks = bbands.perf_tracker.handle_simulation_end()
            report.collect(stockid, results, risks)
            print "%s pass" % (stockid)
        except:
            print traceback.format_exc()
            continue

    if report.report.empty:
        return

    # report summary
    stream = report.summary(dtype='html')
    report.write(stream, 'bbands.html')

    for stockid in report.iter_symbol():
        stream = report.iter_report(stockid, dtype='html')
        report.write(stream, "bbands_%s.html" % (stockid))

    for stockid in report.iter_symbol():
        perf = report.pool[stockid]
        dates = [date2num(i) for i in perf.index[maxlen:]]
        quotes = [
            perf[label][maxlen:].values
            for label in ['open', 'high', 'low', 'close']
        ]
        quotes = zip(*([dates] + quotes))

        fig = plt.figure(facecolor='#07000d')

        ax1 = plt.subplot2grid((6, 4), (1, 0),
                               rowspan=4,
                               colspan=4,
                               axisbg='#07000d')
        candlestick_ohlc(ax1,
                         quotes,
                         width=.6,
                         colorup='#53c156',
                         colordown='#ff1717')

        ax1.plot(dates,
                 perf['upper'][maxlen:].values,
                 '#e1edf9',
                 label='upper',
                 linewidth=1.5)
        ax1.plot(dates,
                 perf['middle'][maxlen:].values,
                 '#e1edf9',
                 label='middle',
                 linewidth=1.5)
        ax1.plot(dates,
                 perf['lower'][maxlen:].values,
                 '#e1edf9',
                 label='lower',
                 linewidth=1.5)

        ax1.grid(True, color='w')
        ax1.xaxis.set_major_locator(mticker.MaxNLocator(10))
        ax1.xaxis.set_major_formatter(DateFormatter('%Y-%m-%d'))
        ax1.yaxis.label.set_color("w")
        ax1.spines['bottom'].set_color("#5998ff")
        ax1.spines['top'].set_color("#5998ff")
        ax1.spines['left'].set_color("#5998ff")
        ax1.spines['right'].set_color("#5998ff")
        ax1.tick_params(axis='y', colors='w')
        plt.gca().yaxis.set_major_locator(mticker.MaxNLocator(prune='upper'))
        ax1.tick_params(axis='x', colors='w')
        plt.ylabel('Stock price and Volume')

        bbLeg = plt.legend(loc=9,
                           ncol=2,
                           prop={'size': 7},
                           fancybox=True,
                           borderaxespad=0.)
        bbLeg.get_frame().set_alpha(0.4)
        textEd = pylab.gca().get_legend().get_texts()
        pylab.setp(textEd[0:6], color='w')

        ax1v = ax1.twinx()
        ax1v.fill_between(dates,
                          0,
                          perf['volume'][maxlen:].values,
                          facecolor='#00ffe8',
                          alpha=.4)
        ax1v.axes.yaxis.set_ticklabels([])
        ax1v.grid(False)
        ###Edit this to 3, so it's a bit larger
        ax1v.set_ylim(0, 3 * perf['volume'][maxlen:].values.max())
        ax1v.spines['bottom'].set_color("#5998ff")
        ax1v.spines['top'].set_color("#5998ff")
        ax1v.spines['left'].set_color("#5998ff")
        ax1v.spines['right'].set_color("#5998ff")
        ax1v.tick_params(axis='x', colors='w')
        ax1v.tick_params(axis='y', colors='w')

        plt.setp(ax1.get_xticklabels(), visible=False)
        plt.subplots_adjust(left=.09,
                            bottom=.14,
                            right=.94,
                            top=.95,
                            wspace=.20,
                            hspace=0)
        plt.gcf().set_size_inches(18, 8)
        plt.savefig("bbands_%s.png" % (stockid), facecolor=fig.get_facecolor())
Example #11
0
def _run(
    handle_data,
    initialize,
    before_trading_start,
    analyze,
    algofile,
    algotext,
    defines,
    data_frequency,
    capital_base,
    bundle,
    bundle_timestamp,
    start,
    end,
    output,
    trading_calendar,
    print_algo,
    metrics_set,
    local_namespace,
    environ,
    blotter,
    custom_loader,
    benchmark_spec,
):
    """Run a backtest for the given algorithm.

    This is shared between the cli and :func:`zipline.run_algo`.
    """

    bundle_data = bundles.load(
        bundle,
        environ,
        bundle_timestamp,
    )

    if trading_calendar is None:
        trading_calendar = get_calendar("XNYS")

    # date parameter validation
    if trading_calendar.session_distance(start, end) < 1:
        raise _RunAlgoError(
            "There are no trading days between %s and %s" % (
                start.date(),
                end.date(),
            ), )

    benchmark_sid, benchmark_returns = benchmark_spec.resolve(
        asset_finder=bundle_data.asset_finder,
        start_date=start,
        end_date=end,
    )

    if algotext is not None:
        if local_namespace:
            ip = get_ipython()  # noqa
            namespace = ip.user_ns
        else:
            namespace = {}

        for assign in defines:
            try:
                name, value = assign.split("=", 2)
            except ValueError:
                raise ValueError(
                    "invalid define %r, should be of the form name=value" %
                    assign, )
            try:
                # evaluate in the same namespace so names may refer to
                # eachother
                namespace[name] = eval(value, namespace)
            except Exception as e:
                raise ValueError(
                    "failed to execute definition for name %r: %s" %
                    (name, e), )
    elif defines:
        raise _RunAlgoError(
            "cannot pass define without `algotext`",
            "cannot pass '-D' / '--define' without '-t' / '--algotext'",
        )
    else:
        namespace = {}
        if algofile is not None:
            algotext = algofile.read()

    if print_algo:
        if PYGMENTS:
            highlight(
                algotext,
                PythonLexer(),
                TerminalFormatter(),
                outfile=sys.stdout,
            )
        else:
            click.echo(algotext)

    first_trading_day = bundle_data.equity_minute_bar_reader.first_trading_day

    data = DataPortal(
        bundle_data.asset_finder,
        trading_calendar=trading_calendar,
        first_trading_day=first_trading_day,
        equity_minute_reader=bundle_data.equity_minute_bar_reader,
        equity_daily_reader=bundle_data.equity_daily_bar_reader,
        adjustment_reader=bundle_data.adjustment_reader,
        future_minute_reader=bundle_data.equity_minute_bar_reader,
        future_daily_reader=bundle_data.equity_daily_bar_reader,
    )

    pipeline_loader = USEquityPricingLoader.without_fx(
        bundle_data.equity_daily_bar_reader,
        bundle_data.adjustment_reader,
    )

    def choose_loader(column):
        if column in USEquityPricing.columns:
            return pipeline_loader
        try:
            return custom_loader.get(column)
        except KeyError:
            raise ValueError("No PipelineLoader registered for column %s." %
                             column)

    if isinstance(metrics_set, str):
        try:
            metrics_set = metrics.load(metrics_set)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    if isinstance(blotter, str):
        try:
            blotter = load(Blotter, blotter)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    try:
        perf = TradingAlgorithm(
            namespace=namespace,
            data_portal=data,
            get_pipeline_loader=choose_loader,
            trading_calendar=trading_calendar,
            sim_params=SimulationParameters(
                start_session=start,
                end_session=end,
                trading_calendar=trading_calendar,
                capital_base=capital_base,
                data_frequency=data_frequency,
            ),
            metrics_set=metrics_set,
            blotter=blotter,
            benchmark_returns=benchmark_returns,
            benchmark_sid=benchmark_sid,
            **{
                "initialize": initialize,
                "handle_data": handle_data,
                "before_trading_start": before_trading_start,
                "analyze": analyze,
            } if algotext is None else {
                "algo_filename": getattr(algofile, "name", "<algorithm>"),
                "script": algotext,
            },
        ).run()
    except NoBenchmark:
        raise _RunAlgoError(
            ("No ``benchmark_spec`` was provided, and"
             " ``zipline.api.set_benchmark`` was not called in"
             " ``initialize``."),
            ("Neither '--benchmark-symbol' nor '--benchmark-sid' was"
             " provided, and ``zipline.api.set_benchmark`` was not called"
             " in ``initialize``. Did you mean to pass '--no-benchmark'?"),
        )

    if output == "-":
        click.echo(str(perf))
    elif output != os.devnull:  # make the zipline magic not write any data
        perf.to_pickle(output)

    return perf
Example #12
0
def run(opt='twse', debug=False, limit=0):
    """ as doctest run """
    maxlen = 30
    starttime = datetime.utcnow() - timedelta(days=300)
    endtime = datetime.utcnow()
    report = Report('dualema',
                    sort=[('buys', False), ('sells', False),
                          ('portfolio_value', False)],
                    limit=20)

    kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
    # fetch
    idhandler = TwseIdDBHandler(
        **kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
    for stockid in idhandler.stock.get_ids():
        try:
            kwargs = {
                'opt': opt,
                'targets': ['stock'],
                'starttime': starttime,
                'endtime': endtime,
                'stockids': [stockid],
                'traderids': [],
                'base': 'stock',
                'callback': None,
                'limit': 1,
                'debug': debug
            }
            panel, dbhandler = collect_hisframe(**kwargs)
            if len(panel[stockid].index) < maxlen:
                continue

            sim_params = SimulationParameters(
                period_start=panel[stockid].index[0],
                period_end=panel[stockid].index[-1],
                data_frequency='daily',
                emission_rate='daily')

            dualema = DualEMA(dbhandler=dbhandler,
                              debug=debug,
                              sim_params=sim_params)
            results = dualema.run(panel).fillna(0)
            risks = dualema.perf_tracker.handle_simulation_end()
            report.collect(stockid, results, risks)
            print "%s pass" % (stockid)
        except:
            print traceback.format_exc()
            continue

    if report.report.empty:
        return

    # report summary
    stream = report.summary(dtype='html')
    report.write(stream, 'dualema.html')

    for stockid in report.iter_symbol():
        stream = report.iter_report(stockid, dtype='html')
        report.write(stream, "dualema_%s.html" % (stockid))

    # plot
    for stockid in report.iter_symbol():
        try:
            perf = report.pool[stockid]
            fig = plt.figure()
            ax1 = fig.add_subplot(211, ylabel='portfolio value')
            perf.portfolio_value.plot(ax=ax1)

            ax2 = fig.add_subplot(212)
            perf[['short_ema', 'long_ema']].plot(ax=ax2)

            ax2.plot(perf.ix[perf.buy].index,
                     perf.short_ema[perf.buy],
                     '^',
                     markersize=10,
                     color='m')
            ax2.plot(perf.ix[perf.sell].index,
                     perf.short_ema[perf.sell],
                     'v',
                     markersize=10,
                     color='k')
            plt.legend(loc=0)
            plt.gcf().set_size_inches(18, 8)
            plt.savefig("dualema_%s.png" % (stockid))
            #plt.show()
        except:
            continue
Example #13
0
    def test_tracker(self, parameter_comment, days_to_delete):
        """
        @days_to_delete - configures which days in the data set we should
        remove, used for ensuring that we still return performance messages
        even when there is no data.
        """
        # This date range covers Columbus day,
        # however Columbus day is not a market holiday
        #
        #     October 2008
        # Su Mo Tu We Th Fr Sa
        #           1  2  3  4
        #  5  6  7  8  9 10 11
        # 12 13 14 15 16 17 18
        # 19 20 21 22 23 24 25
        # 26 27 28 29 30 31
        start_dt = datetime.datetime(year=2008,
                                     month=10,
                                     day=9,
                                     tzinfo=pytz.utc)
        end_dt = datetime.datetime(year=2008,
                                   month=10,
                                   day=16,
                                   tzinfo=pytz.utc)

        trade_count = 6
        sid = 133
        price = 10.1
        price_list = [price] * trade_count
        volume = [100] * trade_count
        trade_time_increment = datetime.timedelta(days=1)

        sim_params = SimulationParameters(period_start=start_dt,
                                          period_end=end_dt)

        trade_history = factory.create_trade_history(sid,
                                                     price_list,
                                                     volume,
                                                     trade_time_increment,
                                                     sim_params,
                                                     source_id="factory1")

        sid2 = 134
        price2 = 12.12
        price2_list = [price2] * trade_count
        trade_history2 = factory.create_trade_history(sid2,
                                                      price2_list,
                                                      volume,
                                                      trade_time_increment,
                                                      sim_params,
                                                      source_id="factory2")
        # 'middle' start of 3 depends on number of days == 7
        middle = 3

        # First delete from middle
        if days_to_delete.middle:
            del trade_history[middle:(middle + days_to_delete.middle)]
            del trade_history2[middle:(middle + days_to_delete.middle)]

        # Delete start
        if days_to_delete.start:
            del trade_history[:days_to_delete.start]
            del trade_history2[:days_to_delete.start]

        # Delete from end
        if days_to_delete.end:
            del trade_history[-days_to_delete.end:]
            del trade_history2[-days_to_delete.end:]

        sim_params.first_open = \
            sim_params.calculate_first_open()
        sim_params.last_close = \
            sim_params.calculate_last_close()
        sim_params.capital_base = 1000.0
        sim_params.frame_index = ['sid', 'volume', 'dt', 'price', 'changed']
        perf_tracker = perf.PerformanceTracker(sim_params)

        events = date_sorted_sources(trade_history, trade_history2)

        events = [
            self.event_with_txn(event, trade_history[0].dt) for event in events
        ]

        # Extract events with transactions to use for verification.
        events_with_txns = [event for event in events if event.TRANSACTION]

        perf_messages = \
            [msg for date, snapshot in
             perf_tracker.transform(
                 itertools.groupby(events, attrgetter('dt')))
             for event in snapshot
             for msg in event.perf_messages]

        end_perf_messages, risk_message = perf_tracker.handle_simulation_end()

        perf_messages.extend(end_perf_messages)

        #we skip two trades, to test case of None transaction
        self.assertEqual(perf_tracker.txn_count, len(events_with_txns))

        cumulative_pos = perf_tracker.cumulative_performance.positions[sid]
        expected_size = len(events_with_txns) / 2 * -25
        self.assertEqual(cumulative_pos.amount, expected_size)

        self.assertEqual(perf_tracker.last_close,
                         perf_tracker.cumulative_risk_metrics.end_date)

        self.assertEqual(len(perf_messages), sim_params.days_in_period)
Example #14
0
    def run(self,
            algo,
            tBeg=None,
            tEnd=None,
            commission=None,
            slippage=None,
            warn=False,
            bar_source='yahoo',
            adjusted=True,
            data_frequency='daily',
            include_open=True,
            csi_port='ETFs'):

        # Set a default algo if none is provided
        if not algo:
            raise Exception('No algo provided')
        self.algo = algo

        # set commission model
        if commission:
            self.algo.set_commission(commission)
        else:
            self.algo.set_commission(
                PerShareWithMin(comm_per_share=0.01, comm_min=1.0))

        # set slippage model
        if slippage:
            self.algo.set_slippage(slippage)
        else:
            self.algo.set_slippage(FixedSlippage(spread=0.0))

        # guess starting and ending dates if none are provided
        if not tEnd:
            tEnd = get_end_date()
        if not tBeg:
            tBeg = get_start_date(self.algo.iL, tNow=tEnd)

        tBeg = pytz.utc.localize(tBeg)
        tEnd = pytz.utc.localize(tEnd)

        self.sim_params = SimulationParameters(tBeg,
                                               tEnd,
                                               data_frequency=data_frequency,
                                               capital_base=self.capital_base,
                                               make_new_environment=True,
                                               extra_dates=[])
        # print self.sim_params

        source = self.get_bar_source(tBeg,
                                     tEnd,
                                     bar_source,
                                     adjusted=adjusted,
                                     include_open=include_open,
                                     csi_port=csi_port)

        if bar_source == 'redshift':
            bench_source, self.bench_price_utc = redshift.get_bench_source(
                tBeg, tEnd)
        else:
            bench_source = None

        sources = [source]

        if not warn:
            # turn off warnings
            import warnings
            warnings.filterwarnings('ignore')

        self.run_time = time.time()
        self.results = self.algo.run(sources,
                                     sim_params=self.sim_params,
                                     benchmark_return_source=bench_source)
        self.run_time = time.time() - self.run_time
Example #15
0
    def test_tracker(self, parameter_comment, days_to_delete):
        """
        @days_to_delete - configures which days in the data set we should
        remove, used for ensuring that we still return performance messages
        even when there is no data.
        """
        # This date range covers Columbus day,
        # however Columbus day is not a market holiday
        #
        #     October 2008
        # Su Mo Tu We Th Fr Sa
        #           1  2  3  4
        #  5  6  7  8  9 10 11
        # 12 13 14 15 16 17 18
        # 19 20 21 22 23 24 25
        # 26 27 28 29 30 31
        start_dt = datetime.datetime(year=2008,
                                     month=10,
                                     day=9,
                                     tzinfo=pytz.utc)
        end_dt = datetime.datetime(year=2008,
                                   month=10,
                                   day=16,
                                   tzinfo=pytz.utc)

        trade_count = 6
        sid = 133
        price = 10.1
        price_list = [price] * trade_count
        volume = [100] * trade_count
        trade_time_increment = datetime.timedelta(days=1)

        sim_params = SimulationParameters(
            period_start=start_dt,
            period_end=end_dt
        )

        benchmark_events = benchmark_events_in_range(sim_params)

        trade_history = factory.create_trade_history(
            sid,
            price_list,
            volume,
            trade_time_increment,
            sim_params,
            source_id="factory1"
        )

        sid2 = 134
        price2 = 12.12
        price2_list = [price2] * trade_count
        trade_history2 = factory.create_trade_history(
            sid2,
            price2_list,
            volume,
            trade_time_increment,
            sim_params,
            source_id="factory2"
        )
        # 'middle' start of 3 depends on number of days == 7
        middle = 3

        # First delete from middle
        if days_to_delete.middle:
            del trade_history[middle:(middle + days_to_delete.middle)]
            del trade_history2[middle:(middle + days_to_delete.middle)]

        # Delete start
        if days_to_delete.start:
            del trade_history[:days_to_delete.start]
            del trade_history2[:days_to_delete.start]

        # Delete from end
        if days_to_delete.end:
            del trade_history[-days_to_delete.end:]
            del trade_history2[-days_to_delete.end:]

        sim_params.first_open = \
            sim_params.calculate_first_open()
        sim_params.last_close = \
            sim_params.calculate_last_close()
        sim_params.capital_base = 1000.0
        sim_params.frame_index = [
            'sid',
            'volume',
            'dt',
            'price',
            'changed']
        perf_tracker = perf.PerformanceTracker(
            sim_params
        )

        events = date_sorted_sources(trade_history, trade_history2)

        events = [event for event in
                  self.trades_with_txns(events, trade_history[0].dt)]

        # Extract events with transactions to use for verification.
        txns = [event for event in
                events if event.type == DATASOURCE_TYPE.TRANSACTION]

        orders = [event for event in
                  events if event.type == DATASOURCE_TYPE.ORDER]

        all_events = date_sorted_sources(events, benchmark_events)

        filtered_events = [filt_event for filt_event
                           in all_events if filt_event.dt <= end_dt]
        filtered_events.sort(key=lambda x: x.dt)
        grouped_events = itertools.groupby(filtered_events, lambda x: x.dt)
        perf_messages = []

        for date, group in grouped_events:
            for event in group:
                perf_tracker.process_event(event)
            msg = perf_tracker.handle_market_close()
            perf_messages.append(msg)

        self.assertEqual(perf_tracker.txn_count, len(txns))
        self.assertEqual(perf_tracker.txn_count, len(orders))

        cumulative_pos = perf_tracker.cumulative_performance.positions[sid]
        expected_size = len(txns) / 2 * -25
        self.assertEqual(cumulative_pos.amount, expected_size)

        self.assertEqual(len(perf_messages),
                         sim_params.days_in_period)
Example #16
0
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
         algotext, defines, data_frequency, capital_base, bundle,
         bundle_timestamp, start, end, output, trading_calendar, print_algo,
         metrics_set, local_namespace, environ, blotter, benchmark_returns):
    """Run a backtest for the given algorithm.

    This is shared between the cli and :func:`zipline.run_algo`.
    """
    if benchmark_returns is None:
        benchmark_returns = load_benchmark_data(environ=environ)

    if algotext is not None:
        if local_namespace:
            ip = get_ipython()  # noqa
            namespace = ip.user_ns
        else:
            namespace = {}

        for assign in defines:
            try:
                name, value = assign.split('=', 2)
            except ValueError:
                raise ValueError(
                    'invalid define %r, should be of the form name=value' %
                    assign, )
            try:
                # evaluate in the same namespace so names may refer to
                # eachother
                namespace[name] = eval(value, namespace)
            except Exception as e:
                raise ValueError(
                    'failed to execute definition for name %r: %s' %
                    (name, e), )
    elif defines:
        raise _RunAlgoError(
            'cannot pass define without `algotext`',
            "cannot pass '-D' / '--define' without '-t' / '--algotext'",
        )
    else:
        namespace = {}
        if algofile is not None:
            algotext = algofile.read()

    if print_algo:
        if PYGMENTS:
            highlight(
                algotext,
                PythonLexer(),
                TerminalFormatter(),
                outfile=sys.stdout,
            )
        else:
            click.echo(algotext)

    if trading_calendar is None:
        trading_calendar = get_calendar('XNYS')

    # date parameter validation
    if trading_calendar.session_distance(start, end) < 1:
        raise _RunAlgoError(
            'There are no trading days between %s and %s' % (
                start.date(),
                end.date(),
            ), )

    bundle_data = bundles.load(
        bundle,
        environ,
        bundle_timestamp,
    )

    first_trading_day = \
        bundle_data.equity_minute_bar_reader.first_trading_day

    data = DataPortal(
        bundle_data.asset_finder,
        trading_calendar=trading_calendar,
        first_trading_day=first_trading_day,
        equity_minute_reader=bundle_data.equity_minute_bar_reader,
        equity_daily_reader=bundle_data.equity_daily_bar_reader,
        adjustment_reader=bundle_data.adjustment_reader,
    )

    pipeline_loader = USEquityPricingLoader.without_fx(
        bundle_data.equity_daily_bar_reader,
        bundle_data.adjustment_reader,
    )

    def choose_loader(column):
        if column in USEquityPricing.columns:
            return pipeline_loader
        raise ValueError("No PipelineLoader registered for column %s." %
                         column)

    if isinstance(metrics_set, six.string_types):
        try:
            metrics_set = metrics.load(metrics_set)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    if isinstance(blotter, six.string_types):
        try:
            blotter = load(Blotter, blotter)
        except ValueError as e:
            raise _RunAlgoError(str(e))

    perf = TradingAlgorithm(
        namespace=namespace,
        data_portal=data,
        get_pipeline_loader=choose_loader,
        trading_calendar=trading_calendar,
        sim_params=SimulationParameters(
            start_session=start,
            end_session=end,
            trading_calendar=trading_calendar,
            capital_base=capital_base,
            data_frequency=data_frequency,
            # fix for minute perf
            emission_rate=data_frequency),
        metrics_set=metrics_set,
        blotter=blotter,
        benchmark_returns=benchmark_returns,
        **{
            'initialize': initialize,
            'handle_data': handle_data,
            'before_trading_start': before_trading_start,
            'analyze': analyze,
        } if algotext is None else {
            'algo_filename': getattr(algofile, 'name', '<algorithm>'),
            'script': algotext,
        }).run()

    if output == '-':
        click.echo(str(perf))
    elif output != os.devnull:  # make the zipline magic not write any data
        perf.to_pickle(output)

    return perf
Example #17
0
    def test_minute_tracker(self):
        """ Tests minute performance tracking."""
        with trading.TradingEnvironment():
            start_dt = trading.environment.exchange_dt_in_utc(
                datetime.datetime(2013, 3, 1, 9, 31))
            end_dt = trading.environment.exchange_dt_in_utc(
                datetime.datetime(2013, 3, 1, 16, 0))

            sim_params = SimulationParameters(
                period_start=start_dt,
                period_end=end_dt,
                emission_rate='minute'
            )
            tracker = perf.PerformanceTracker(sim_params)

            foo_event_1 = factory.create_trade('foo', 10.0, 20, start_dt)
            order_event_1 = Order(sid=foo_event_1.sid,
                                  amount=-25,
                                  dt=foo_event_1.dt)
            bar_event_1 = factory.create_trade('bar', 100.0, 200, start_dt)
            txn_event_1 = Transaction(sid=foo_event_1.sid,
                                      amount=-25,
                                      dt=foo_event_1.dt,
                                      price=10.0,
                                      commission=0.50,
                                      order_id=order_event_1.id)
            benchmark_event_1 = Event({
                'dt': start_dt,
                'returns': 0.01,
                'type': DATASOURCE_TYPE.BENCHMARK
            })

            foo_event_2 = factory.create_trade(
                'foo', 11.0, 20, start_dt + datetime.timedelta(minutes=1))
            bar_event_2 = factory.create_trade(
                'bar', 11.0, 20, start_dt + datetime.timedelta(minutes=1))
            benchmark_event_2 = Event({
                'dt': start_dt + datetime.timedelta(minutes=1),
                'returns': 0.02,
                'type': DATASOURCE_TYPE.BENCHMARK
            })

            events = [
                foo_event_1,
                order_event_1,
                benchmark_event_1,
                txn_event_1,
                bar_event_1,
                foo_event_2,
                benchmark_event_2,
                bar_event_2,
            ]

            grouped_events = itertools.groupby(
                events, operator.attrgetter('dt'))

            messages = {}
            for date, group in grouped_events:
                tracker.set_date(date)
                for event in group:
                    tracker.process_event(event)
                tracker.handle_minute_close(date)
                msg = tracker.to_dict()
                messages[date] = msg

            self.assertEquals(2, len(messages))

            msg_1 = messages[foo_event_1.dt]
            msg_2 = messages[foo_event_2.dt]

            self.assertEquals(1, len(msg_1['minute_perf']['transactions']),
                              "The first message should contain one "
                              "transaction.")
            # Check that transactions aren't emitted for previous events.
            self.assertEquals(0, len(msg_2['minute_perf']['transactions']),
                              "The second message should have no "
                              "transactions.")

            self.assertEquals(1, len(msg_1['minute_perf']['orders']),
                              "The first message should contain one orders.")
            # Check that orders aren't emitted for previous events.
            self.assertEquals(0, len(msg_2['minute_perf']['orders']),
                              "The second message should have no orders.")

            # Ensure that period_close moves through time.
            # Also, ensure that the period_closes are the expected dts.
            self.assertEquals(foo_event_1.dt,
                              msg_1['minute_perf']['period_close'])
            self.assertEquals(foo_event_2.dt,
                              msg_2['minute_perf']['period_close'])

            # Ensure that a Sharpe value for cumulative metrics is being
            # created.
            self.assertIsNotNone(msg_1['cumulative_risk_metrics']['sharpe'])
            self.assertIsNotNone(msg_2['cumulative_risk_metrics']['sharpe'])
    def test_old_new_data_api_paths(self):
        """
        Test that the new and old data APIs hit the same code paths.

        We want to ensure that the old data API(data[sid(N)].field and
        similar)  and the new data API(data.current(sid(N), field) and
        similar) hit the same code paths on the DataPortal.
        """
        test_start_minute = self.trading_calendar.minutes_for_session(
            self.sim_params.sessions[0])[1]
        test_end_minute = self.trading_calendar.minutes_for_session(
            self.sim_params.sessions[0])[-1]
        bar_data = self.create_bardata(lambda: test_end_minute, )
        ohlcvp_fields = [
            "open",
            "high",
            "low"
            "close",
            "volume",
            "price",
        ]
        spot_value_meth = 'zipline.data.data_portal.DataPortal.get_spot_value'

        def assert_get_spot_value_called(fun, field):
            """
            Assert that get_spot_value was called during the execution of fun.

            Takes in a function fun and a string field.
            """
            with patch(spot_value_meth) as gsv:
                fun()
                gsv.assert_called_with(self.asset1, field, test_end_minute,
                                       'minute')

        # Ensure that data.current(sid(n), field) has the same behaviour as
        # data[sid(n)].field.
        for field in ohlcvp_fields:
            assert_get_spot_value_called(
                lambda: getattr(bar_data[self.asset1], field),
                field,
            )
            assert_get_spot_value_called(
                lambda: bar_data.current(self.asset1, field),
                field,
            )

        history_meth = 'zipline.data.data_portal.DataPortal.get_history_window'

        def assert_get_history_window_called(fun, is_legacy):
            """
            Assert that get_history_window was called during fun().

            Takes in a function fun and a boolean is_legacy.
            """
            with patch(history_meth) as ghw:
                fun()
                # Slightly hacky, but done to get around the fact that
                # history( explicitly passes an ffill param as the last arg,
                # while data.history doesn't.
                if is_legacy:
                    ghw.assert_called_with(
                        [self.asset1, self.asset2, self.asset3],
                        test_end_minute, 5, "1m", "volume", "minute", True)
                else:
                    ghw.assert_called_with(
                        [self.asset1, self.asset2, self.asset3],
                        test_end_minute,
                        5,
                        "1m",
                        "volume",
                        "minute",
                    )

        test_sim_params = SimulationParameters(
            start_session=test_start_minute,
            end_session=test_end_minute,
            data_frequency="minute",
            trading_calendar=self.trading_calendar,
        )

        history_algorithm = self.create_algo(history_algo,
                                             sim_params=test_sim_params)
        assert_get_history_window_called(lambda: history_algorithm.run(),
                                         is_legacy=True)
        assert_get_history_window_called(lambda: bar_data.history(
            [self.asset1, self.asset2, self.asset3], "volume", 5, "1m"),
                                         is_legacy=False)
Example #19
0
    def test_history_passed_to_talib(self):
        """
        Had an issue where MagicMock was causing errors during validation
        with talib.

        We don't officially support a talib integration, yet.
        But using talib directly should work.
        """
        algo_text = """
import talib
import numpy as np

from zipline.api import history, add_history, record

def initialize(context):
    add_history(2, '1d', 'price')

def handle_data(context, data):
    prices = history(2, '1d', 'price')

    ma_result = talib.MA(np.asarray(prices[0]), timeperiod=2)
    record(ma=ma_result[-1])
""".strip()

        #      April 2007
        # Su Mo Tu We Th Fr Sa
        #  1  2  3  4  5  6  7
        #  8  9 10 11 12 13 14
        # 15 16 17 18 19 20 21
        # 22 23 24 25 26 27 28
        # 29 30

        # Eddie: this was set to 04-10 but I don't see how that makes
        # sense as it does not generate enough data to get at -2 index
        # below.
        start = pd.Timestamp('2007-04-05', tz='UTC')
        end = pd.Timestamp('2007-04-10', tz='UTC')

        sim_params = SimulationParameters(
            period_start=start,
            period_end=end,
            capital_base=float("1.0e5"),
            data_frequency='minute',
            emission_rate='daily'
        )

        test_algo = TradingAlgorithm(
            script=algo_text,
            data_frequency='minute',
            sim_params=sim_params
        )

        source = RandomWalkSource(start=start,
                                  end=end)
        output = test_algo.run(source)
        # At this point, just ensure that there is no crash.
        self.assertIsNotNone(output)

        recorded_ma = output.ix[-2, 'ma']

        self.assertFalse(pd.isnull(recorded_ma))
        # Depends on seed
        np.testing.assert_almost_equal(recorded_ma,
                                       159.76304468946876)
    def test_simple_transforms(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("ignore", PerformanceWarning)
            warnings.simplefilter("default", ZiplineDeprecationWarning)

            sim_params = SimulationParameters(
                start_session=self.sim_params.sessions[8],
                end_session=self.sim_params.sessions[-1],
                data_frequency="minute",
                trading_calendar=self.trading_calendar,
            )

            algo = self.create_algo(simple_transforms_algo,
                                    sim_params=sim_params)
            algo.run()

            self.assertEqual(8, len(w))
            transforms = ["mavg", "vwap", "stddev", "returns"]

            for idx, line_no in enumerate(range(8, 12)):
                warning1 = w[idx * 2]
                warning2 = w[(idx * 2) + 1]

                self.assertEqual("<string>", warning1.filename)
                self.assertEqual("<string>", warning2.filename)

                self.assertEqual(line_no, warning1.lineno)
                self.assertEqual(line_no, warning2.lineno)

                self.assertEqual(
                    "`data[sid(N)]` is deprecated. Use "
                    "`data.current`.", str(warning1.message))
                self.assertEqual(
                    "The `{0}` method is "
                    "deprecated.".format(transforms[idx]),
                    str(warning2.message))

            # now verify the transform values
            # minute price
            # 2016-01-11 14:31:00+00:00    1561
            # ...
            # 2016-01-14 20:59:00+00:00    3119
            # 2016-01-14 21:00:00+00:00    3120
            # 2016-01-15 14:31:00+00:00    3121
            # 2016-01-15 14:32:00+00:00    3122
            # 2016-01-15 14:33:00+00:00    3123

            # volume
            # 2016-01-11 14:31:00+00:00    156100
            # ...
            # 2016-01-14 20:59:00+00:00    311900
            # 2016-01-14 21:00:00+00:00    312000
            # 2016-01-15 14:31:00+00:00    312100
            # 2016-01-15 14:32:00+00:00    312200
            # 2016-01-15 14:33:00+00:00    312300

            # daily price (last day built with minute data)
            # 2016-01-14 00:00:00+00:00       9
            # 2016-01-15 00:00:00+00:00    3123

            # mavg = average of all the prices = (1561 + 3123) / 2 = 2342
            # vwap = sum(price * volume) / sum(volumes)
            #      = 889119531400.0 / 366054600.0
            #      = 2428.9259891830343
            # stddev = stddev(price, ddof=1) = 451.3435498597493
            # returns = (todayprice - yesterdayprice) / yesterdayprice
            #         = (3123 - 9) / 9 = 346
            self.assertEqual(2342, algo.mavg)
            self.assertAlmostEqual(2428.92599, algo.vwap, places=5)
            self.assertAlmostEqual(451.34355, algo.stddev, places=5)
            self.assertAlmostEqual(346, algo.returns)
Example #21
0
    def test_tracker(self, parameter_comment, days_to_delete):
        """
        @days_to_delete - configures which days in the data set we should
        remove, used for ensuring that we still return performance messages
        even when there is no data.
        """
        # This date range covers Columbus day,
        # however Columbus day is not a market holiday
        #
        #     October 2008
        # Su Mo Tu We Th Fr Sa
        #           1  2  3  4
        #  5  6  7  8  9 10 11
        # 12 13 14 15 16 17 18
        # 19 20 21 22 23 24 25
        # 26 27 28 29 30 31
        start_dt = datetime(year=2008,
                            month=10,
                            day=9,
                            tzinfo=pytz.utc)
        end_dt = datetime(year=2008,
                          month=10,
                          day=16,
                          tzinfo=pytz.utc)

        trade_count = 6
        sid = 133
        price = 10.1
        price_list = [price] * trade_count
        volume = [100] * trade_count
        trade_time_increment = timedelta(days=1)

        sim_params = SimulationParameters(
            period_start=start_dt,
            period_end=end_dt
        )

        benchmark_events = benchmark_events_in_range(sim_params)

        trade_history = factory.create_trade_history(
            sid,
            price_list,
            volume,
            trade_time_increment,
            sim_params,
            source_id="factory1"
        )

        sid2 = 134
        price2 = 12.12
        price2_list = [price2] * trade_count
        trade_history2 = factory.create_trade_history(
            sid2,
            price2_list,
            volume,
            trade_time_increment,
            sim_params,
            source_id="factory2"
        )
        # 'middle' start of 3 depends on number of days == 7
        middle = 3

        # First delete from middle
        if days_to_delete.middle:
            del trade_history[middle:(middle + days_to_delete.middle)]
            del trade_history2[middle:(middle + days_to_delete.middle)]

        # Delete start
        if days_to_delete.start:
            del trade_history[:days_to_delete.start]
            del trade_history2[:days_to_delete.start]

        # Delete from end
        if days_to_delete.end:
            del trade_history[-days_to_delete.end:]
            del trade_history2[-days_to_delete.end:]

        sim_params.first_open = \
            sim_params.calculate_first_open()
        sim_params.last_close = \
            sim_params.calculate_last_close()
        sim_params.capital_base = 1000.0
        sim_params.frame_index = [
            'sid',
            'volume',
            'dt',
            'price',
            'changed']
        perf_tracker = perf.PerformanceTracker(
            sim_params
        )

        events = date_sorted_sources(trade_history, trade_history2)

        events = [event for event in
                  self.trades_with_txns(events, trade_history[0].dt)]

        # Extract events with transactions to use for verification.
        txns = [event for event in
                events if event.type == zp.DATASOURCE_TYPE.TRANSACTION]

        orders = [event for event in
                  events if event.type == zp.DATASOURCE_TYPE.ORDER]

        all_events = date_sorted_sources(events, benchmark_events)

        filtered_events = [filt_event for filt_event
                           in all_events if filt_event.dt <= end_dt]
        filtered_events.sort(key=lambda x: x.dt)
        grouped_events = itertools.groupby(filtered_events, lambda x: x.dt)
        perf_messages = []

        for date, group in grouped_events:
            for event in group:
                perf_tracker.process_event(event)
            msg = perf_tracker.handle_market_close_daily()
            perf_messages.append(msg)

        self.assertEqual(perf_tracker.txn_count, len(txns))
        self.assertEqual(perf_tracker.txn_count, len(orders))

        cumulative_pos = perf_tracker.cumulative_performance.positions[sid]
        expected_size = len(txns) / 2 * -25
        self.assertEqual(cumulative_pos.amount, expected_size)

        self.assertEqual(len(perf_messages),
                         sim_params.days_in_period)
Example #22
0
def run(opt='twse', debug=False, limit=0):
    """ as doctest run """
    maxlen = 5
    starttime = datetime.utcnow() - timedelta(days=15)
    endtime = datetime.utcnow()
    report = Report('besttrader',
                    sort=[('buys', False), ('sells', False),
                          ('portfolio_value', False)],
                    limit=20)

    kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
    # 1590:u'花旗環球', 1440:u'美林'
    traderid = '1440'
    idhandler = TwseIdDBHandler(
        **kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
    for stockid in idhandler.stock.get_ids():
        try:
            # pre find traderid as top0
            kwargs = {
                'opt':
                opt,
                'targets': ['trader'],
                'starttime':
                starttime,
                'endtime':
                endtime,
                'stockids': [stockid],
                'traderids': [],
                'base':
                'stock',
                'constraint':
                lambda x: x.value["ebuyratio"] > 10 or x.value["totalkeepbuy"]
                >= 1,
                'order':
                lambda x: [-x.value["totalvolume"], -x.value["totalbuyratio"]],
                'callback':
                None,
                'limit':
                10,
                'debug':
                debug
            }
            panel, dbhandler = collect_hisframe(**kwargs)
            tops = list(
                dbhandler.trader.get_alias([stockid], 'trader',
                                           ["top%d" % i for i in range(10)]))
            if not tops:
                continue

            print "%s prefound:%s" % (stockid, tops)
            traderid = tops[0] if traderid not in tops else traderid
            # run
            kwargs = {
                'opt': opt,
                'targets': ['stock', 'trader', 'future', 'credit'],
                'starttime': starttime,
                'endtime': endtime,
                'stockids': [stockid],
                'traderids': [traderid],
                'base': 'trader',
                'callback': None,
                'limit': 10,
                'debug': debug
            }
            panel, dbhandler = collect_hisframe(**kwargs)
            if len(panel[stockid].index) < maxlen:
                continue

            sim_params = SimulationParameters(
                period_start=panel[stockid].index[0],
                period_end=panel[stockid].index[-1],
                data_frequency='daily',
                emission_rate='daily')

            besttrader = BestTrader(dbhandler=dbhandler,
                                    debug=debug,
                                    sim_params=sim_params)
            results = besttrader.run(panel).fillna(0)
            risks = besttrader.perf_tracker.handle_simulation_end()
            report.collect(stockid, results, risks)
            print "%s pass" % (stockid)
        except:
            print traceback.format_exc()
            continue

    if report.report.empty:
        return

    # report summary
    stream = report.summary(dtype='html')
    report.write(stream, 'besttrader.html')

    for stockid in report.iter_symbol():
        stream = report.iter_report(stockid, dtype='html')
        report.write(stream, "besttrader_%s.html" % (stockid))

    # plot
    for stockid in report.iter_symbol():
        try:
            perf = report.pool[stockid]
            fig = plt.figure()
            ax1 = fig.add_subplot(211, ylabel='portfolio value')
            perf.portfolio_value.plot(ax=ax1)

            ax2 = fig.add_subplot(212)
            perf[['close']].plot(ax=ax2)

            ax2.plot(perf.ix[perf.buy].index,
                     perf.close[perf.buy],
                     '^',
                     markersize=10,
                     color='m')
            ax2.plot(perf.ix[perf.sell].index,
                     perf.close[perf.sell],
                     'v',
                     markersize=10,
                     color='k')
            plt.legend(loc=0)
            plt.gcf().set_size_inches(18, 8)
            plt.savefig("besttrader_%s_%s.png" % (traderid, stockid))
            #plt.show()
        except:
            continue