def init_class_fixtures(cls): super(SecurityListTestCase, cls).init_class_fixtures() # this is ugly, but we need to create two different # TradingEnvironment/DataPortal pairs cls.start = pd.Timestamp(list(LEVERAGED_ETFS.keys())[0]) end = pd.Timestamp('2015-02-17', tz='utc') cls.extra_knowledge_date = pd.Timestamp('2015-01-27', tz='utc') cls.trading_day_before_first_kd = pd.Timestamp('2015-01-23', tz='utc') symbols = ['AAPL', 'GOOG', 'BZQ', 'URTY', 'JFT'] cls.env = cls.enter_class_context( tmp_trading_env( equities=pd.DataFrame.from_records([{ 'start_date': cls.start, 'end_date': end, 'symbol': symbol, 'exchange': "TEST", } for symbol in symbols]), load=cls.make_load_function(), )) cls.sim_params = factory.create_simulation_parameters( start=cls.start, num_days=4, trading_calendar=cls.trading_calendar) cls.sim_params2 = sp2 = factory.create_simulation_parameters( start=cls.trading_day_before_first_kd, num_days=4) cls.env2 = cls.enter_class_context( tmp_trading_env( equities=pd.DataFrame.from_records([{ 'start_date': sp2.start_session, 'end_date': sp2.end_session, 'symbol': symbol, 'exchange': "TEST", } for symbol in symbols]), load=cls.make_load_function(), )) cls.tempdir = cls.enter_class_context(tmp_dir()) cls.tempdir2 = cls.enter_class_context(tmp_dir()) cls.data_portal = create_data_portal( asset_finder=cls.env.asset_finder, tempdir=cls.tempdir, sim_params=cls.sim_params, sids=range(0, 5), trading_calendar=cls.trading_calendar, ) cls.data_portal2 = create_data_portal( asset_finder=cls.env2.asset_finder, tempdir=cls.tempdir2, sim_params=cls.sim_params2, sids=range(0, 5), trading_calendar=cls.trading_calendar, )
def test_fetcher_bad_data(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/fetcher_nflx_data.csv', body=NFLX_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2013-06-12", tz='UTC'), end=pd.Timestamp("2013-06-14", tz='UTC'), data_frequency="minute" ) results = self.run_algo(""" from gateway.api import fetch_csv, symbol import numpy as np def initialize(context): fetch_csv('https://fake.urls.com/fetcher_nflx_data.csv', date_column = 'Settlement Date', date_format = '%m/%d/%y') context.nflx = symbol('NFLX') context.aapl = symbol('AAPL') def handle_data(context, data): assert np.isnan(data.current(context.nflx, 'invalid_column')) assert np.isnan(data.current(context.aapl, 'invalid_column')) assert np.isnan(data.current(context.aapl, 'dtc')) """, sim_params=sim_params, data_frequency="minute") self.assertEqual(3, len(results))
def test_fetcher_in_before_trading_start(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/fetcher_nflx_data.csv', body=NFLX_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2013-06-13", tz='UTC'), end=pd.Timestamp("2013-11-15", tz='UTC'), data_frequency="minute" ) results = self.run_algo(""" from gateway.api import fetch_csv, record, symbol def initialize(context): fetch_csv('https://fake.urls.com/fetcher_nflx_data.csv', date_column = 'Settlement Date', date_format = '%m/%d/%y') context.stock = symbol('NFLX') def before_trading_start(context, data): record(Short_Interest = data.current(context.stock, 'dtc')) """, sim_params=sim_params, data_frequency="minute") values = results["Short_Interest"] np.testing.assert_array_equal(values[0:33], np.full(33, np.nan)) np.testing.assert_array_almost_equal(values[33:44], [1.690317] * 11) np.testing.assert_array_almost_equal(values[44:55], [2.811858] * 11) np.testing.assert_array_almost_equal(values[55:64], [2.50233] * 9) np.testing.assert_array_almost_equal(values[64:75], [2.550829] * 11) np.testing.assert_array_almost_equal(values[75:], [2.64484] * 35)
def test_fetcher_universe_non_security_return(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/bad_fetcher_universe_data.csv', body=NON_ASSET_FETCHER_UNIVERSE_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-09", tz='UTC'), end=pd.Timestamp("2006-01-10", tz='UTC') ) self.run_algo( """ from gateway.api import fetch_csv def initialize(context): fetch_csv( 'https://fake.urls.com/bad_fetcher_universe_data.csv', date_format='%m/%d/%Y' ) def handle_data(context, data): if len(data.fetcher_assets) > 0: raise Exception("Shouldn't be any assets in fetcher_assets!") """, sim_params=sim_params, )
def test_minutely_fetcher(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/aapl_minute_csv_data.csv', body=AAPL_MINUTE_CSV_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-03", tz='UTC'), end=pd.Timestamp("2006-01-10", tz='UTC'), emission_rate="minute", data_frequency="minute" ) test_algo = TradingAlgorithm( script=""" from gateway.api import fetch_csv, record, sid def initialize(context): fetch_csv('https://fake.urls.com/aapl_minute_csv_data.csv') def handle_data(context, data): record(aapl_signal=data.current(sid(24), "signal")) """, sim_params=sim_params, data_frequency="minute", env=self.env) # manually setting data portal and getting generator because we need # the minutely emission packets here. TradingAlgorithm.run() only # returns daily packets. test_algo.data_portal = FetcherDataPortal(self.env, self.trading_calendar) gen = test_algo.get_generator() perf_packets = list(gen) signal = [result["minute_perf"]["recorded_vars"]["aapl_signal"] for result in perf_packets if "minute_perf" in result] self.assertEqual(6 * 390, len(signal)) # csv data is: # symbol,date,signal # aapl,1/4/06 5:31AM, 1 # aapl,1/4/06 11:30AM, 2 # aapl,1/5/06 5:31AM, 1 # aapl,1/5/06 11:30AM, 3 # aapl,1/9/06 5:31AM, 1 # aapl,1/9/06 11:30AM, 4 for dates 1/3 to 1/10 # 2 signals per day, only last signal is taken. So we expect # 390 bars of signal NaN on 1/3 # 390 bars of signal 2 on 1/4 # 390 bars of signal 3 on 1/5 # 390 bars of signal 3 on 1/6 (forward filled) # 390 bars of signal 4 on 1/9 # 390 bars of signal 4 on 1/9 (forward filled) np.testing.assert_array_equal([np.NaN] * 390, signal[0:390]) np.testing.assert_array_equal([2] * 390, signal[390:780]) np.testing.assert_array_equal([3] * 780, signal[780:1560]) np.testing.assert_array_equal([4] * 780, signal[1560:])
def test_before_trading_start(self, test_name, num_days, freq, emission_rate): params = factory.create_simulation_parameters( num_days=num_days, data_frequency=freq, emission_rate=emission_rate) def fake_benchmark(self, dt): return 0.01 with patch.object(BenchmarkSource, "get_value", self.fake_minutely_benchmark): algo = BeforeTradingAlgorithm(sim_params=params, env=self.env) algo.run(FakeDataPortal(self.env)) self.assertEqual( len(algo.perf_tracker.sim_params.sessions), num_days ) bts_minutes = days_at_time( params.sessions, time(8, 45), "US/Eastern" ) self.assertTrue( bts_minutes.equals( pd.DatetimeIndex(algo.before_trading_at) ), "Expected %s but was %s." % (params.sessions, algo.before_trading_at))
def test_algo_without_rl_violation_after_delete(self): sim_params = factory.create_simulation_parameters( start=self.extra_knowledge_date, num_days=4, ) equities = pd.DataFrame.from_records([{ 'symbol': 'BZQ', 'start_date': sim_params.start_session, 'end_date': sim_params.end_session, 'exchange': "TEST", }]) with TempDirectory() as new_tempdir, \ security_list_copy(), \ tmp_trading_env(equities=equities, load=self.make_load_function()) as env: # add a delete statement removing bzq # write a new delete statement file to disk add_security_data([], ['BZQ']) data_portal = create_data_portal( env.asset_finder, new_tempdir, sim_params, range(0, 5), trading_calendar=self.trading_calendar, ) algo = RestrictedAlgoWithoutCheck(symbol='BZQ', sim_params=sim_params, env=env) algo.run(data_portal)
def test_minutely_emissions_generate_performance_stats_for_last_day(self): params = factory.create_simulation_parameters(num_days=1, data_frequency='minute', emission_rate='minute') with patch.object(BenchmarkSource, "get_value", self.fake_minutely_benchmark): algo = NoopAlgorithm(sim_params=params, env=self.env) algo.run(FakeDataPortal(self.env)) self.assertEqual(len(algo.perf_tracker.sim_params.sessions), 1)
def test_fetcher_universe(self, name, data, column_name): # Patching fetch_url here rather than using responses because (a) it's # easier given the paramaterization, and (b) there are enough tests # using responses that the fetch_url code is getting a good workout so # we don't have to use it in every test. with patch('gateway.sources.requests_csv.PandasRequestsCSV.fetch_url', new=lambda *a, **k: data): sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-09", tz='UTC'), end=pd.Timestamp("2006-01-11", tz='UTC') ) algocode = """ from pandas import Timestamp from pandas.tseries.tools import normalize_date from gateway.api import fetch_csv, record, sid, get_datetime def initialize(context): fetch_csv( 'https://dl.dropbox.com/u/16705795/dtoc_history.csv', date_format='%m/%d/%Y'{token} ) context.expected_sids = {{ Timestamp('2006-01-09 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-10 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-11 00:00:00+0000', tz='UTC'):[24, 3766, 5061, 14848] }} context.bar_count = 0 def handle_data(context, data): expected = context.expected_sids[normalize_date(get_datetime())] actual = data.fetcher_assets for stk in expected: if stk not in actual: raise Exception( "{{stk}} is missing on dt={{dt}}".format( stk=stk, dt=get_datetime())) record(sid_count=len(actual)) record(bar_count=context.bar_count) context.bar_count += 1 """ replacement = "" if column_name: replacement = ",symbol_column='%s'\n" % column_name real_algocode = algocode.format(token=replacement) results = self.run_algo(real_algocode, sim_params=sim_params) self.assertEqual(len(results), 3) self.assertEqual(3, results["sid_count"].iloc[0]) self.assertEqual(3, results["sid_count"].iloc[1]) self.assertEqual(4, results["sid_count"].iloc[2])
def test_fetcher_universe_minute(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/fetcher_universe_data.csv', body=FETCHER_UNIVERSE_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-09", tz='UTC'), end=pd.Timestamp("2006-01-11", tz='UTC'), data_frequency="minute" ) results = self.run_algo( """ from pandas import Timestamp from gateway.api import fetch_csv, record, get_datetime def initialize(context): fetch_csv( 'https://fake.urls.com/fetcher_universe_data.csv', date_format='%m/%d/%Y' ) context.expected_sids = { Timestamp('2006-01-09 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-10 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-11 00:00:00+0000', tz='UTC'):[24, 3766, 5061, 14848] } context.bar_count = 0 def handle_data(context, data): expected = context.expected_sids[get_datetime().replace(hour=0, minute=0)] actual = data.fetcher_assets for stk in expected: if stk not in actual: raise Exception("{stk} is missing".format(stk=stk)) record(sid_count=len(actual)) record(bar_count=context.bar_count) context.bar_count += 1 """, sim_params=sim_params, data_frequency="minute" ) self.assertEqual(3, len(results)) self.assertEqual(3, results["sid_count"].iloc[0]) self.assertEqual(3, results["sid_count"].iloc[1]) self.assertEqual(4, results["sid_count"].iloc[2])
def test_algo_with_rl_violation_cumulative(self): """ Add a new restriction, run a test long after both knowledge dates, make sure stock from original restriction set is still disallowed. """ sim_params = factory.create_simulation_parameters(start=self.start + timedelta(days=7), num_days=4) with security_list_copy(): add_security_data(['AAPL'], []) algo = RestrictedAlgoWithoutCheck(symbol='BZQ', sim_params=sim_params, env=self.env) with self.assertRaises(TradingControlViolation) as ctx: algo.run(self.data_portal) self.check_algo_exception(algo, ctx, 0)
def test_algo_with_rl_violation_after_knowledge_date(self): sim_params = factory.create_simulation_parameters(start=self.start + timedelta(days=7), num_days=5) data_portal = create_data_portal( self.env.asset_finder, self.tempdir, sim_params=sim_params, sids=range(0, 5), trading_calendar=self.trading_calendar, ) algo = RestrictedAlgoWithoutCheck(symbol='BZQ', sim_params=sim_params, env=self.env) with self.assertRaises(TradingControlViolation) as ctx: algo.run(data_portal) self.check_algo_exception(algo, ctx, 0)
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, trading_calendar, print_algo, local_namespace, environ): """Run a backtest for the given algorithm. This is shared between the cli and :func:`gateway.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) if trading_calendar is None: trading_calendar = get_calendar('NYSE') if bundle is not None: bundle_data = load( bundle, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment(asset_db_path=connstr, environ=environ) first_trading_day =\ bundle_data.equity_minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, trading_calendar=trading_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) pipeline_loader = USEquityPricingLoader( bundle_data.equity_daily_bar_reader, bundle_data.adjustment_reader, ) def choose_loader(column): if column in USEquityPricing.columns: return pipeline_loader raise ValueError("No PipelineLoader registered for column %s." % column) else: env = TradingEnvironment(environ=environ) choose_loader = None perf = TradingAlgorithm( namespace=namespace, env=env, get_pipeline_loader=choose_loader, trading_calendar=trading_calendar, sim_params=create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, trading_calendar=trading_calendar, ), **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the gateway magic not write any data perf.to_pickle(output) return perf
def create_test_gateway(**config): """ :param config: A configuration object that is a dict with: - sid - an integer, which will be used as the asset ID. - order_count - the number of orders the test algo will place, defaults to 100 - order_amount - the number of shares per order, defaults to 100 - trade_count - the number of trades to simulate, defaults to 101 to ensure all orders are processed. - algorithm - optional parameter providing an algorithm. defaults to :py:class:`gateway.test.algorithms.TestAlgorithm` - trade_source - optional parameter to specify trades, if present. If not present :py:class:`gateway.sources.SpecificEquityTrades` is the source, with daily frequency in trades. - slippage: optional parameter that configures the :py:class:`gateway.gens.tradingsimulation.TransactionSimulator`. Expects an object with a simulate mehod, such as :py:class:`gateway.gens.tradingsimulation.FixedSlippage`. :py:mod:`gateway.finance.trading` """ assert isinstance(config, dict) try: sid_list = config['sid_list'] except KeyError: try: sid_list = [config['sid']] except KeyError: raise Exception("simfactory create_test_gateway() requires " "argument 'sid_list' or 'sid'") concurrent_trades = config.get('concurrent_trades', False) order_count = config.get('order_count', 100) order_amount = config.get('order_amount', 100) trading_calendar = config.get('trading_calendar', get_calendar("NYSE")) # ------------------- # Create the Algo # ------------------- if 'algorithm' in config: test_algo = config['algorithm'] else: test_algo = TestAlgorithm(sid_list[0], order_amount, order_count, sim_params=config.get( 'sim_params', factory.create_simulation_parameters()), trading_calendar=trading_calendar, slippage=config.get('slippage'), identifiers=sid_list) # ------------------- # Trade Source # ------------------- if 'skip_data' not in config: if 'trade_source' in config: trade_source = config['trade_source'] else: trade_source = factory.create_daily_trade_source( sid_list, test_algo.sim_params, test_algo.trading_environment, trading_calendar, concurrent=concurrent_trades, ) trades_by_sid = {} for trade in trade_source: if trade.sid not in trades_by_sid: trades_by_sid[trade.sid] = [] trades_by_sid[trade.sid].append(trade) data_portal = create_data_portal_from_trade_history( config['env'].asset_finder, trading_calendar, config['tempdir'], config['sim_params'], trades_by_sid) test_algo.data_portal = data_portal # ------------------- # Benchmark source # ------------------- test_algo.benchmark_return_source = config.get('benchmark_source', None) # ------------------ # generator/simulator sim = test_algo.get_generator() return sim
def transaction_sim(self, **params): """This is a utility method that asserts expected results for conversion of orders to transactions given a trade history """ trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') asset1 = self.asset_finder.retrieve_asset(1) metadata = make_simple_equity_info([asset1.sid], self.start, self.end) with TempDirectory() as tempdir, \ tmp_trading_env(equities=metadata, load=self.make_load_function()) as env: if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( start=self.start, end=self.end, data_frequency="minute") minutes = self.trading_calendar.minutes_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { asset1.sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( self.trading_calendar, self.trading_calendar.sessions_in_range( self.trading_calendar.minute_to_session_label( minutes[0]), self.trading_calendar.minute_to_session_label( minutes[-1])), tempdir.path, iteritems(assets), ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily") days = sim_params.sessions assets = { 1: pd.DataFrame( { "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") BcolzDailyBarWriter(path, self.trading_calendar, days[0], days[-1]).write(assets.items()) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, equity_daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedSlippage() else: slippage_func = None blotter = Blotter(sim_params.data_frequency, slippage_func) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = PerformanceTracker(sim_params, self.trading_calendar, self.env) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator**len(order_list) order_id = blotter.order(asset1, order_amount * direction, MarketOrder()) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData( data_portal=data_portal, simulation_dt_func=lambda: tick, data_frequency=sim_params.data_frequency, trading_calendar=self.trading_calendar, restrictions=NoRestrictions(), ) txns, _, closed_orders = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) blotter.prune_orders(closed_orders) for i in range(order_count): order = order_list[i] self.assertEqual(order.asset, asset1) self.assertEqual(order.amount, order_amount * alternator**i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) cumulative_pos = tracker.position_tracker.positions[asset1] if total_volume == 0: self.assertIsNone(cumulative_pos) else: self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain the asset. oo = blotter.open_orders self.assertNotIn(asset1, oo, "Entry is removed when no open orders")