def init_class_fixtures(cls): super(SecurityListTestCase, cls).init_class_fixtures() # this is ugly, but we need to create two different # TradingEnvironment/DataPortal pairs cls.start = pd.Timestamp(list(LEVERAGED_ETFS.keys())[0]) end = pd.Timestamp('2015-02-17', tz='utc') cls.extra_knowledge_date = pd.Timestamp('2015-01-27', tz='utc') cls.trading_day_before_first_kd = pd.Timestamp('2015-01-23', tz='utc') symbols = ['AAPL', 'GOOG', 'BZQ', 'URTY', 'JFT'] cls.env = cls.enter_class_context( tmp_trading_env( equities=pd.DataFrame.from_records([{ 'start_date': cls.start, 'end_date': end, 'symbol': symbol, 'exchange': "TEST", } for symbol in symbols]), load=cls.make_load_function(), )) cls.sim_params = factory.create_simulation_parameters( start=cls.start, num_days=4, trading_calendar=cls.trading_calendar) cls.sim_params2 = sp2 = factory.create_simulation_parameters( start=cls.trading_day_before_first_kd, num_days=4) cls.env2 = cls.enter_class_context( tmp_trading_env( equities=pd.DataFrame.from_records([{ 'start_date': sp2.start_session, 'end_date': sp2.end_session, 'symbol': symbol, 'exchange': "TEST", } for symbol in symbols]), load=cls.make_load_function(), )) cls.tempdir = cls.enter_class_context(tmp_dir()) cls.tempdir2 = cls.enter_class_context(tmp_dir()) cls.data_portal = create_data_portal( asset_finder=cls.env.asset_finder, tempdir=cls.tempdir, sim_params=cls.sim_params, sids=range(0, 5), trading_calendar=cls.trading_calendar, ) cls.data_portal2 = create_data_portal( asset_finder=cls.env2.asset_finder, tempdir=cls.tempdir2, sim_params=cls.sim_params2, sids=range(0, 5), trading_calendar=cls.trading_calendar, )
def test_fetcher_in_before_trading_start(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/fetcher_nflx_data.csv', body=NFLX_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2013-06-13", tz='UTC'), end=pd.Timestamp("2013-11-15", tz='UTC'), data_frequency="minute" ) results = self.run_algo(""" from catalyst.api import fetch_csv, record, symbol def initialize(context): fetch_csv('https://fake.urls.com/fetcher_nflx_data.csv', date_column = 'Settlement Date', date_format = '%m/%d/%y') context.stock = symbol('NFLX') def before_trading_start(context, data): record(Short_Interest = data.current(context.stock, 'dtc')) """, sim_params=sim_params, data_frequency="minute") values = results["Short_Interest"] np.testing.assert_array_equal(values[0:33], np.full(33, np.nan)) np.testing.assert_array_almost_equal(values[33:44], [1.690317] * 11) np.testing.assert_array_almost_equal(values[44:55], [2.811858] * 11) np.testing.assert_array_almost_equal(values[55:64], [2.50233] * 9) np.testing.assert_array_almost_equal(values[64:75], [2.550829] * 11) np.testing.assert_array_almost_equal(values[75:], [2.64484] * 35)
def _test_before_trading_start(self, test_name, num_days, freq, emission_rate): params = factory.create_simulation_parameters( num_days=num_days, data_frequency=freq, emission_rate=emission_rate) def fake_benchmark(self, dt): return 0.01 with patch.object(BenchmarkSource, "get_value", self.fake_minutely_benchmark): algo = BeforeTradingAlgorithm(sim_params=params, env=self.env) algo.run(FakeDataPortal(self.env)) self.assertEqual(len(algo.perf_tracker.sim_params.sessions), num_days) bts_minutes = days_at_time(params.sessions, time(8, 45), "US/Eastern") self.assertTrue( bts_minutes.equals(pd.DatetimeIndex(algo.before_trading_at)), "Expected %s but was %s." % (params.sessions, algo.before_trading_at))
def test_fetcher_universe_non_security_return(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/bad_fetcher_universe_data.csv', body=NON_ASSET_FETCHER_UNIVERSE_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-09", tz='UTC'), end=pd.Timestamp("2006-01-10", tz='UTC') ) self.run_algo( """ from catalyst.api import fetch_csv def initialize(context): fetch_csv( 'https://fake.urls.com/bad_fetcher_universe_data.csv', date_format='%m/%d/%Y' ) def handle_data(context, data): if len(data.fetcher_assets) > 0: raise Exception("Shouldn't be any assets in fetcher_assets!") """, sim_params=sim_params, )
def test_fetcher_universe_non_security_return(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/bad_fetcher_universe_data.csv', body=NON_ASSET_FETCHER_UNIVERSE_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-09", tz='UTC'), end=pd.Timestamp("2006-01-10", tz='UTC')) self.run_algo( """ from catalyst.api import fetch_csv def initialize(context): fetch_csv( 'https://fake.urls.com/bad_fetcher_universe_data.csv', date_format='%m/%d/%Y' ) def handle_data(context, data): if len(data.fetcher_assets) > 0: raise Exception("Shouldn't be any assets in fetcher_assets!") """, sim_params=sim_params, )
def test_minutely_fetcher(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/aapl_minute_csv_data.csv', body=AAPL_MINUTE_CSV_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-03", tz='UTC'), end=pd.Timestamp("2006-01-10", tz='UTC'), emission_rate="minute", data_frequency="minute" ) test_algo = TradingAlgorithm( script=""" from catalyst.api import fetch_csv, record, sid def initialize(context): fetch_csv('https://fake.urls.com/aapl_minute_csv_data.csv') def handle_data(context, data): record(aapl_signal=data.current(sid(24), "signal")) """, sim_params=sim_params, data_frequency="minute", env=self.env) # manually setting data portal and getting generator because we need # the minutely emission packets here. TradingAlgorithm.run() only # returns daily packets. test_algo.data_portal = FetcherDataPortal(self.env, self.trading_calendar) gen = test_algo.get_generator() perf_packets = list(gen) signal = [result["minute_perf"]["recorded_vars"]["aapl_signal"] for result in perf_packets if "minute_perf" in result] self.assertEqual(6 * 390, len(signal)) # csv data is: # symbol,date,signal # aapl,1/4/06 5:31AM, 1 # aapl,1/4/06 11:30AM, 2 # aapl,1/5/06 5:31AM, 1 # aapl,1/5/06 11:30AM, 3 # aapl,1/9/06 5:31AM, 1 # aapl,1/9/06 11:30AM, 4 for dates 1/3 to 1/10 # 2 signals per day, only last signal is taken. So we expect # 390 bars of signal NaN on 1/3 # 390 bars of signal 2 on 1/4 # 390 bars of signal 3 on 1/5 # 390 bars of signal 3 on 1/6 (forward filled) # 390 bars of signal 4 on 1/9 # 390 bars of signal 4 on 1/9 (forward filled) np.testing.assert_array_equal([np.NaN] * 390, signal[0:390]) np.testing.assert_array_equal([2] * 390, signal[390:780]) np.testing.assert_array_equal([3] * 780, signal[780:1560]) np.testing.assert_array_equal([4] * 780, signal[1560:])
def test_fetcher_in_before_trading_start(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/fetcher_nflx_data.csv', body=NFLX_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2013-06-13", tz='UTC'), end=pd.Timestamp("2013-11-15", tz='UTC'), data_frequency="minute") results = self.run_algo(""" from catalyst.api import fetch_csv, record, symbol def initialize(context): fetch_csv('https://fake.urls.com/fetcher_nflx_data.csv', date_column = 'Settlement Date', date_format = '%m/%d/%y') context.stock = symbol('NFLX') def before_trading_start(context, data): record(Short_Interest = data.current(context.stock, 'dtc')) """, sim_params=sim_params, data_frequency="minute") values = results["Short_Interest"] np.testing.assert_array_equal(values[0:33], np.full(33, np.nan)) np.testing.assert_array_almost_equal(values[33:44], [1.690317] * 11) np.testing.assert_array_almost_equal(values[44:55], [2.811858] * 11) np.testing.assert_array_almost_equal(values[55:64], [2.50233] * 9) np.testing.assert_array_almost_equal(values[64:75], [2.550829] * 11) np.testing.assert_array_almost_equal(values[75:], [2.64484] * 35)
def test_algo_without_rl_violation_after_delete(self): sim_params = factory.create_simulation_parameters( start=self.extra_knowledge_date, num_days=4, ) equities = pd.DataFrame.from_records([{ 'symbol': 'BZQ', 'start_date': sim_params.start_session, 'end_date': sim_params.end_session, 'exchange': "TEST", }]) with TempDirectory() as new_tempdir, \ security_list_copy(), \ tmp_trading_env(equities=equities, load=self.make_load_function()) as env: # add a delete statement removing bzq # write a new delete statement file to disk add_security_data([], ['BZQ']) data_portal = create_data_portal( env.asset_finder, new_tempdir, sim_params, range(0, 5), trading_calendar=self.trading_calendar, ) algo = RestrictedAlgoWithoutCheck(symbol='BZQ', sim_params=sim_params, env=env) algo.run(data_portal)
def test_fetcher_bad_data(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/fetcher_nflx_data.csv', body=NFLX_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2013-06-12", tz='UTC'), end=pd.Timestamp("2013-06-14", tz='UTC'), data_frequency="minute") results = self.run_algo(""" from catalyst.api import fetch_csv, symbol import numpy as np def initialize(context): fetch_csv('https://fake.urls.com/fetcher_nflx_data.csv', date_column = 'Settlement Date', date_format = '%m/%d/%y') context.nflx = symbol('NFLX') context.aapl = symbol('AAPL') def handle_data(context, data): assert np.isnan(data.current(context.nflx, 'invalid_column')) assert np.isnan(data.current(context.aapl, 'invalid_column')) assert np.isnan(data.current(context.aapl, 'dtc')) """, sim_params=sim_params, data_frequency="minute") self.assertEqual(3, len(results))
def test_fetcher_bad_data(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/fetcher_nflx_data.csv', body=NFLX_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2013-06-12", tz='UTC'), end=pd.Timestamp("2013-06-14", tz='UTC'), data_frequency="minute" ) results = self.run_algo(""" from catalyst.api import fetch_csv, symbol import numpy as np def initialize(context): fetch_csv('https://fake.urls.com/fetcher_nflx_data.csv', date_column = 'Settlement Date', date_format = '%m/%d/%y') context.nflx = symbol('NFLX') context.aapl = symbol('AAPL') def handle_data(context, data): assert np.isnan(data.current(context.nflx, 'invalid_column')) assert np.isnan(data.current(context.aapl, 'invalid_column')) assert np.isnan(data.current(context.aapl, 'dtc')) """, sim_params=sim_params, data_frequency="minute") self.assertEqual(3, len(results))
def test_before_trading_start(self, test_name, num_days, freq, emission_rate): params = factory.create_simulation_parameters( num_days=num_days, data_frequency=freq, emission_rate=emission_rate) def fake_benchmark(self, dt): return 0.01 with patch.object(BenchmarkSource, "get_value", self.fake_minutely_benchmark): algo = BeforeTradingAlgorithm(sim_params=params, env=self.env) algo.run(FakeDataPortal(self.env)) self.assertEqual( len(algo.perf_tracker.sim_params.sessions), num_days ) bts_minutes = days_at_time( params.sessions, time(8, 45), "US/Eastern" ) self.assertTrue( bts_minutes.equals( pd.DatetimeIndex(algo.before_trading_at) ), "Expected %s but was %s." % (params.sessions, algo.before_trading_at))
def test_minutely_emissions_generate_performance_stats_for_last_day(self): params = factory.create_simulation_parameters(num_days=1, data_frequency='minute', emission_rate='minute') with patch.object(BenchmarkSource, "get_value", self.fake_minutely_benchmark): algo = NoopAlgorithm(sim_params=params, env=self.env) algo.run(FakeDataPortal(self.env)) self.assertEqual(len(algo.perf_tracker.sim_params.sessions), 1)
def test_fetcher_universe(self, name, data, column_name): # Patching fetch_url here rather than using responses because (a) it's # easier given the paramaterization, and (b) there are enough tests # using responses that the fetch_url code is getting a good workout so # we don't have to use it in every test. with patch('catalyst.sources.requests_csv.PandasRequestsCSV.fetch_url', new=lambda *a, **k: data): sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-09", tz='UTC'), end=pd.Timestamp("2006-01-11", tz='UTC') ) algocode = """ from pandas import Timestamp from pandas.tseries.tools import normalize_date from catalyst.api import fetch_csv, record, sid, get_datetime def initialize(context): fetch_csv( 'https://dl.dropbox.com/u/16705795/dtoc_history.csv', date_format='%m/%d/%Y'{token} ) context.expected_sids = {{ Timestamp('2006-01-09 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-10 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-11 00:00:00+0000', tz='UTC'):[24, 3766, 5061, 14848] }} context.bar_count = 0 def handle_data(context, data): expected = context.expected_sids[normalize_date(get_datetime())] actual = data.fetcher_assets for stk in expected: if stk not in actual: raise Exception( "{{stk}} is missing on dt={{dt}}".format( stk=stk, dt=get_datetime())) record(sid_count=len(actual)) record(bar_count=context.bar_count) context.bar_count += 1 """ replacement = "" if column_name: replacement = ",symbol_column='%s'\n" % column_name real_algocode = algocode.format(token=replacement) results = self.run_algo(real_algocode, sim_params=sim_params) self.assertEqual(len(results), 3) self.assertEqual(3, results["sid_count"].iloc[0]) self.assertEqual(3, results["sid_count"].iloc[1]) self.assertEqual(4, results["sid_count"].iloc[2])
def test_fetcher_universe(self, name, data, column_name): # Patching fetch_url here rather than using responses because (a) it's # easier given the paramaterization, and (b) there are enough tests # using responses that the fetch_url code is getting a good workout so # we don't have to use it in every test. with patch('catalyst.sources.requests_csv.PandasRequestsCSV.fetch_url', new=lambda *a, **k: data): sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-09", tz='UTC'), end=pd.Timestamp("2006-01-11", tz='UTC')) algocode = """ from pandas import Timestamp from catalyst.api import fetch_csv, record, sid, get_datetime from catalyst.utils.compat import normalize_date def initialize(context): fetch_csv( 'https://dl.dropbox.com/u/16705795/dtoc_history.csv', date_format='%m/%d/%Y'{token} ) context.expected_sids = {{ Timestamp('2006-01-09 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-10 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-11 00:00:00+0000', tz='UTC'):[24, 3766, 5061, 14848] }} context.bar_count = 0 def handle_data(context, data): expected = context.expected_sids[normalize_date(get_datetime())] actual = data.fetcher_assets for stk in expected: if stk not in actual: raise Exception( "{{stk}} is missing on dt={{dt}}".format( stk=stk, dt=get_datetime())) record(sid_count=len(actual)) record(bar_count=context.bar_count) context.bar_count += 1 """ replacement = "" if column_name: replacement = ",symbol_column='%s'\n" % column_name real_algocode = algocode.format(token=replacement) results = self.run_algo(real_algocode, sim_params=sim_params) self.assertEqual(len(results), 3) self.assertEqual(3, results["sid_count"].iloc[0]) self.assertEqual(3, results["sid_count"].iloc[1]) self.assertEqual(4, results["sid_count"].iloc[2])
def test_fetcher_universe_minute(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/fetcher_universe_data.csv', body=FETCHER_UNIVERSE_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-09", tz='UTC'), end=pd.Timestamp("2006-01-11", tz='UTC'), data_frequency="minute" ) results = self.run_algo( """ from pandas import Timestamp from catalyst.api import fetch_csv, record, get_datetime def initialize(context): fetch_csv( 'https://fake.urls.com/fetcher_universe_data.csv', date_format='%m/%d/%Y' ) context.expected_sids = { Timestamp('2006-01-09 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-10 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-11 00:00:00+0000', tz='UTC'):[24, 3766, 5061, 14848] } context.bar_count = 0 def handle_data(context, data): expected = context.expected_sids[get_datetime().replace(hour=0, minute=0)] actual = data.fetcher_assets for stk in expected: if stk not in actual: raise Exception("{stk} is missing".format(stk=stk)) record(sid_count=len(actual)) record(bar_count=context.bar_count) context.bar_count += 1 """, sim_params=sim_params, data_frequency="minute" ) self.assertEqual(3, len(results)) self.assertEqual(3, results["sid_count"].iloc[0]) self.assertEqual(3, results["sid_count"].iloc[1]) self.assertEqual(4, results["sid_count"].iloc[2])
def test_fetcher_universe_minute(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/fetcher_universe_data.csv', body=FETCHER_UNIVERSE_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-09", tz='UTC'), end=pd.Timestamp("2006-01-11", tz='UTC'), data_frequency="minute") results = self.run_algo(""" from pandas import Timestamp from catalyst.api import fetch_csv, record, get_datetime def initialize(context): fetch_csv( 'https://fake.urls.com/fetcher_universe_data.csv', date_format='%m/%d/%Y' ) context.expected_sids = { Timestamp('2006-01-09 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-10 00:00:00+0000', tz='UTC'):[24, 3766, 5061], Timestamp('2006-01-11 00:00:00+0000', tz='UTC'):[24, 3766, 5061, 14848] } context.bar_count = 0 def handle_data(context, data): expected = context.expected_sids[get_datetime().replace(hour=0, minute=0)] actual = data.fetcher_assets for stk in expected: if stk not in actual: raise Exception("{stk} is missing".format(stk=stk)) record(sid_count=len(actual)) record(bar_count=context.bar_count) context.bar_count += 1 """, sim_params=sim_params, data_frequency="minute") self.assertEqual(3, len(results)) self.assertEqual(3, results["sid_count"].iloc[0]) self.assertEqual(3, results["sid_count"].iloc[1]) self.assertEqual(4, results["sid_count"].iloc[2])
def test_algo_with_rl_violation_cumulative(self): """ Add a new restriction, run a test long after both knowledge dates, make sure stock from original restriction set is still disallowed. """ sim_params = factory.create_simulation_parameters(start=self.start + timedelta(days=7), num_days=4) with security_list_copy(): add_security_data(['AAPL'], []) algo = RestrictedAlgoWithoutCheck(symbol='BZQ', sim_params=sim_params, env=self.env) with self.assertRaises(TradingControlViolation) as ctx: algo.run(self.data_portal) self.check_algo_exception(algo, ctx, 0)
def test_algo_with_rl_violation_after_knowledge_date(self): sim_params = factory.create_simulation_parameters(start=self.start + timedelta(days=7), num_days=5) data_portal = create_data_portal( self.env.asset_finder, self.tempdir, sim_params=sim_params, sids=range(0, 5), trading_calendar=self.trading_calendar, ) algo = RestrictedAlgoWithoutCheck(symbol='BZQ', sim_params=sim_params, env=self.env) with self.assertRaises(TradingControlViolation) as ctx: algo.run(data_portal) self.check_algo_exception(algo, ctx, 0)
def transaction_sim(self, **params): """This is a utility method that asserts expected results for conversion of orders to transactions given a trade history """ trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') asset1 = self.asset_finder.retrieve_asset(1) metadata = make_simple_equity_info([asset1.sid], self.start, self.end) with TempDirectory() as tempdir, \ tmp_trading_env(equities=metadata, load=self.make_load_function()) as env: if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( start=self.start, end=self.end, data_frequency="minute" ) minutes = self.trading_calendar.minutes_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { asset1.sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( self.trading_calendar, self.trading_calendar.sessions_in_range( self.trading_calendar.minute_to_session_label( minutes[0] ), self.trading_calendar.minute_to_session_label( minutes[-1] ) ), tempdir.path, iteritems(assets), ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily" ) days = sim_params.sessions assets = { 1: pd.DataFrame({ "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") BcolzDailyBarWriter(path, self.trading_calendar, days[0], days[-1]).write( assets.items() ) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedSlippage() else: slippage_func = None blotter = Blotter(sim_params.data_frequency, slippage_func) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = PerformanceTracker(sim_params, self.trading_calendar, self.env) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator ** len(order_list) order_id = blotter.order( asset1, order_amount * direction, MarketOrder()) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData( data_portal=data_portal, simulation_dt_func=lambda: tick, data_frequency=sim_params.data_frequency, trading_calendar=self.trading_calendar, restrictions=NoRestrictions(), ) txns, _, closed_orders = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) blotter.prune_orders(closed_orders) for i in range(order_count): order = order_list[i] self.assertEqual(order.asset, asset1) self.assertEqual(order.amount, order_amount * alternator ** i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) cumulative_pos = tracker.position_tracker.positions[asset1] if total_volume == 0: self.assertIsNone(cumulative_pos) else: self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain the asset. oo = blotter.open_orders self.assertNotIn( asset1, oo, "Entry is removed when no open orders" )
def create_test_catalyst(**config): """ :param config: A configuration object that is a dict with: - sid - an integer, which will be used as the asset ID. - order_count - the number of orders the test algo will place, defaults to 100 - order_amount - the number of shares per order, defaults to 100 - trade_count - the number of trades to simulate, defaults to 101 to ensure all orders are processed. - algorithm - optional parameter providing an algorithm. defaults to :py:class:`catalyst.test.algorithms.TestAlgorithm` - trade_source - optional parameter to specify trades, if present. If not present :py:class:`catalyst.sources.SpecificEquityTrades` is the source, with daily frequency in trades. - slippage: optional parameter that configures the :py:class:`catalyst.gens.tradingsimulation.TransactionSimulator`. Expects an object with a simulate mehod, such as :py:class:`catalyst.gens.tradingsimulation.FixedSlippage`. :py:mod:`catalyst.finance.trading` """ assert isinstance(config, dict) try: sid_list = config['sid_list'] except KeyError: try: sid_list = [config['sid']] except KeyError: raise Exception("simfactory create_test_catalyst() requires " "argument 'sid_list' or 'sid'") concurrent_trades = config.get('concurrent_trades', False) order_count = config.get('order_count', 100) order_amount = config.get('order_amount', 100) trading_calendar = config.get('trading_calendar', get_calendar("NYSE")) # ------------------- # Create the Algo # ------------------- if 'algorithm' in config: test_algo = config['algorithm'] else: test_algo = TestAlgorithm(sid_list[0], order_amount, order_count, sim_params=config.get( 'sim_params', factory.create_simulation_parameters()), trading_calendar=trading_calendar, slippage=config.get('slippage'), identifiers=sid_list) # ------------------- # Trade Source # ------------------- if 'skip_data' not in config: if 'trade_source' in config: trade_source = config['trade_source'] else: trade_source = factory.create_daily_trade_source( sid_list, test_algo.sim_params, test_algo.trading_environment, trading_calendar, concurrent=concurrent_trades, ) trades_by_sid = {} for trade in trade_source: if trade.sid not in trades_by_sid: trades_by_sid[trade.sid] = [] trades_by_sid[trade.sid].append(trade) data_portal = create_data_portal_from_trade_history( config['env'].asset_finder, trading_calendar, config['tempdir'], config['sim_params'], trades_by_sid) test_algo.data_portal = data_portal # ------------------- # Benchmark source # ------------------- test_algo.benchmark_return_source = config.get('benchmark_source', None) # ------------------ # generator/simulator sim = test_algo.get_generator() return sim
def test_minutely_fetcher(self): self.responses.add( self.responses.GET, 'https://fake.urls.com/aapl_minute_csv_data.csv', body=AAPL_MINUTE_CSV_DATA, content_type='text/csv', ) sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-03", tz='UTC'), end=pd.Timestamp("2006-01-10", tz='UTC'), emission_rate="minute", data_frequency="minute") test_algo = TradingAlgorithm(script=""" from catalyst.api import fetch_csv, record, sid def initialize(context): fetch_csv('https://fake.urls.com/aapl_minute_csv_data.csv') def handle_data(context, data): record(aapl_signal=data.current(sid(24), "signal")) """, sim_params=sim_params, data_frequency="minute", env=self.env) # manually setting data portal and getting generator because we need # the minutely emission packets here. TradingAlgorithm.run() only # returns daily packets. test_algo.data_portal = FetcherDataPortal(self.env, self.trading_calendar) gen = test_algo.get_generator() perf_packets = list(gen) signal = [ result["minute_perf"]["recorded_vars"]["aapl_signal"] for result in perf_packets if "minute_perf" in result ] self.assertEqual(6 * 390, len(signal)) # csv data is: # symbol,date,signal # aapl,1/4/06 5:31AM, 1 # aapl,1/4/06 11:30AM, 2 # aapl,1/5/06 5:31AM, 1 # aapl,1/5/06 11:30AM, 3 # aapl,1/9/06 5:31AM, 1 # aapl,1/9/06 11:30AM, 4 for dates 1/3 to 1/10 # 2 signals per day, only last signal is taken. So we expect # 390 bars of signal NaN on 1/3 # 390 bars of signal 2 on 1/4 # 390 bars of signal 3 on 1/5 # 390 bars of signal 3 on 1/6 (forward filled) # 390 bars of signal 4 on 1/9 # 390 bars of signal 4 on 1/9 (forward filled) np.testing.assert_array_equal([np.NaN] * 390, signal[0:390]) np.testing.assert_array_equal([2] * 390, signal[390:780]) np.testing.assert_array_equal([3] * 780, signal[780:1560]) np.testing.assert_array_equal([4] * 780, signal[1560:])
def _build_algo_and_data(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph, analyze_live, simulate_orders, stats_output): namespace = _build_namespace(algotext, local_namespace, defines) if algotext is not None: algotext = algofile.read() if print_algo: _pretty_print_code(algotext) mode = _mode(simulate_orders, live) log.info('running algo in {mode} mode'.format(mode=mode)) exchanges = _build_exchanges_dict(exchange, live, simulate_orders, base_currency) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ, start_dt=start, end_dt=end), environ=environ, exchange_tz='UTC', asset_db_path=None) # We don't need an asset db, we have exchanges env.asset_finder = ExchangeAssetFinder(exchanges=exchanges) choose_loader = partial(_choose_loader, data_frequency) if live: start, end = _get_live_time_range() data_frequency = 'minute' # TODO double check if this is the desired behavior sim_params = create_simulation_parameters(start=start, end=end, capital_base=capital_base, emission_rate=data_frequency, data_frequency=data_frequency) if algotext is None: algorithm_class_kwargs = { 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze } else: algorithm_class_kwargs = { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext } if live: return _build_live_algo_and_data( sim_params, exchanges, env, open_calendar, simulate_orders, algo_namespace, capital_base, live_graph, stats_output, analyze_live, base_currency, namespace, choose_loader, algorithm_class_kwargs) else: return _build_backtest_algo_and_data(exchanges, bundle, env, environ, bundle_timestamp, open_calendar, start, end, namespace, choose_loader, sim_params, algorithm_class_kwargs)
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, quote_currency, live_graph, analyze_live, simulate_orders, auth_aliases, stats_output): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ # TODO: refactor for more granularity if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) log.info('Catalyst version {}'.format(catalyst.__version__)) if not DISABLE_ALPHA_WARNING: log.warn(ALPHA_WARNING_MESSAGE) # sleep(3) if live: if simulate_orders: mode = 'paper-trading' else: mode = 'live-trading' else: mode = 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') if isinstance(auth_aliases, string_types): aliases = auth_aliases.split(',') if len(aliases) < 2 or len(aliases) % 2 != 0: raise ValueError( 'the `auth_aliases` parameter must contain an even list ' 'of comma-delimited values. For example, ' '"binance,auth2" or "binance,auth2,bittrex,auth2".' ) auth_aliases = dict(zip(aliases[::2], aliases[1::2])) exchange_list = [x.strip().lower() for x in exchange.split(',')] exchanges = dict() for name in exchange_list: if auth_aliases is not None and name in auth_aliases: auth_alias = auth_aliases[name] else: auth_alias = None exchanges[name] = get_exchange( exchange_name=name, quote_currency=quote_currency, must_authenticate=(live and not simulate_orders), skip_init=True, auth_alias=auth_alias, ) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial( load_crypto_market_data, environ=environ, start_dt=start, end_dt=end ), environ=environ, exchange_tz='UTC', asset_db_path=None # We don't need an asset db, we have exchanges ) env.asset_finder = ExchangeAssetFinder(exchanges=exchanges) def choose_loader(column): bound_cols = TradingPairPricing.columns if column in bound_cols: return ExchangePricingLoader(data_frequency) raise ValueError( "No PipelineLoader registered for column %s." % column ) if live: # TODO: fix the start data. # is_start checks if a start date was specified by user # needed for live clock is_start = True if start is None: start = pd.Timestamp.utcnow() is_start = False elif start: assert pd.Timestamp.utcnow() <= start, \ "specified start date is in the past." elif start and end: assert start < end, "start date is later than end date." # TODO: fix the end data. # is_end checks if an end date was specified by user # needed for live clock is_end = True if end is None: end = start + timedelta(hours=8760) is_end = False data = DataPortalExchangeLive( exchanges=exchanges, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime('today', utc=True) ) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, emission_rate='minute', data_frequency='minute' ) # TODO: use the constructor instead sim_params._arena = 'live' algorithm_class = partial( ExchangeTradingAlgorithmLive, exchanges=exchanges, algo_namespace=algo_namespace, live_graph=live_graph, simulate_orders=simulate_orders, stats_output=stats_output, analyze_live=analyze_live, start=start, is_start=is_start, end=end, is_end=is_end, ) elif exchanges: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. # I don't think that we should have arbitrary price data bundles # Instead, we should center this data around exchanges. # We still need to support bundles for other misc data, but we # can handle this later. if (start and start != pd.tslib.normalize_date(start)) or \ (end and end != pd.tslib.normalize_date(end)): # todo: add to Sim_Params the option to # start & end at specific times log.warn( "Catalyst currently starts and ends on the start and " "end of the dates specified, respectively. We hope to " "Modify this and support specific times in a future release." ) data = DataPortalExchangeBacktest( exchange_names=[ex_name for ex_name in exchanges], asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, last_available_session=end ) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) algorithm_class = partial( ExchangeTradingAlgorithmBacktest, exchanges=exchanges ) elif bundle is not None: bundle_data = load( bundle, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment(asset_db_path=connstr, environ=environ) first_trading_day = \ bundle_data.equity_minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) perf = algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, } ).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) if bundle is not None: bundles = bundle.split(',') def get_trading_env_and_data(bundles): env = data = None b = 'poloniex' if len(bundles) == 0: return env, data elif len(bundles) == 1: b = bundles[0] bundle_data = load( b, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ), bm_symbol='USDT_BTC', trading_calendar=open_calendar, asset_db_path=connstr, environ=environ, ) first_trading_day = bundle_data.minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, minute_reader=bundle_data.minute_bar_reader, five_minute_reader=bundle_data.five_minute_bar_reader, daily_reader=bundle_data.daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) return env, data def get_loader_for_bundle(b): bundle_data = load( b, environ, bundle_timestamp, ) if b == 'poloniex': return CryptoPricingLoader( bundle_data, data_frequency, CryptoPricing, ) elif b == 'quandl': return USEquityPricingLoader( bundle_data, data_frequency, USEquityPricing, ) raise ValueError("No PipelineLoader registered for bundle %s." % b) loaders = [get_loader_for_bundle(b) for b in bundles] env, data = get_trading_env_and_data(bundles) def choose_loader(column): for loader in loaders: if column in loader.columns: return loader raise ValueError("No PipelineLoader registered for column %s." % column) else: env = TradingEnvironment(environ=environ) choose_loader = None perf = TradingAlgorithm( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ), **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph, analyze_live, simulate_orders, stats_output): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) mode = 'paper-trading' if simulate_orders else 'live-trading' \ if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') exchange_list = [x.strip().lower() for x in exchange.split(',')] exchanges = dict() for exchange_name in exchange_list: exchanges[exchange_name] = get_exchange( exchange_name=exchange_name, base_currency=base_currency, must_authenticate=(live and not simulate_orders), skip_init=True, ) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ, start_dt=start, end_dt=end), environ=environ, exchange_tz='UTC', asset_db_path=None # We don't need an asset db, we have exchanges ) env.asset_finder = ExchangeAssetFinder(exchanges=exchanges) def choose_loader(column): bound_cols = TradingPairPricing.columns if column in bound_cols: return ExchangePricingLoader(data_frequency) raise ValueError("No PipelineLoader registered for column %s." % column) if live: start = pd.Timestamp.utcnow() # TODO: fix the end data. end = start + timedelta(hours=8760) data = DataPortalExchangeLive(exchanges=exchanges, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime( 'today', utc=True)) def fetch_capital_base(exchange, attempt_index=0): """ Fetch the base currency amount required to bootstrap the algorithm against the exchange. The algorithm cannot continue without this value. :param exchange: the targeted exchange :param attempt_index: :return capital_base: the amount of base currency available for trading """ try: log.debug('retrieving capital base in {} to bootstrap ' 'exchange {}'.format(base_currency, exchange_name)) balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: log.warn('could not retrieve balances on {}: {}'.format( exchange.name, e)) sleep(5) return fetch_capital_base(exchange, attempt_index + 1) else: raise ExchangeRequestErrorTooManyAttempts( attempts=attempt_index, error=e) if base_currency in balances: base_currency_available = balances[base_currency]['free'] log.info( 'base currency available in the account: {} {}'.format( base_currency_available, base_currency)) return base_currency_available else: raise BaseCurrencyNotFoundError(base_currency=base_currency, exchange=exchange_name) if not simulate_orders: for exchange_name in exchanges: exchange = exchanges[exchange_name] balance = fetch_capital_base(exchange) if balance < capital_base: raise NotEnoughCapitalError( exchange=exchange_name, base_currency=base_currency, balance=balance, capital_base=capital_base, ) sim_params = create_simulation_parameters(start=start, end=end, capital_base=capital_base, emission_rate='minute', data_frequency='minute') # TODO: use the constructor instead sim_params._arena = 'live' algorithm_class = partial( ExchangeTradingAlgorithmLive, exchanges=exchanges, algo_namespace=algo_namespace, live_graph=live_graph, simulate_orders=simulate_orders, stats_output=stats_output, analyze_live=analyze_live, ) elif exchanges: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. # I don't think that we should have arbitrary price data bundles # Instead, we should center this data around exchanges. # We still need to support bundles for other misc data, but we # can handle this later. data = DataPortalExchangeBacktest( exchange_names=[exchange_name for exchange_name in exchanges], asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, last_available_session=end) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) algorithm_class = partial(ExchangeTradingAlgorithmBacktest, exchanges=exchanges) elif bundle is not None: bundle_data = load( bundle, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment(asset_db_path=connstr, environ=environ) first_trading_day = \ bundle_data.equity_minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) perf = algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) mode = 'live' if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') exchange_list = [x.strip().lower() for x in exchange.split(',')] exchanges = dict() for exchange_name in exchange_list: # Looking for the portfolio from the cache first portfolio = get_algo_object(algo_name=algo_namespace, key='portfolio_{}'.format(exchange_name), environ=environ) if portfolio is None: portfolio = ExchangePortfolio(start_date=pd.Timestamp.utcnow()) # This corresponds to the json file containing api token info exchange_auth = get_exchange_auth(exchange_name) if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''): raise ExchangeAuthEmpty(exchange=exchange_name.title(), filename=os.path.join( get_exchange_folder( exchange_name, environ), 'auth.json')) if exchange_name == 'bitfinex': exchanges[exchange_name] = Bitfinex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'bittrex': exchanges[exchange_name] = Bittrex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'poloniex': exchanges[exchange_name] = Poloniex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) else: raise ExchangeNotFoundError(exchange_name=exchange_name) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ, start_dt=start, end_dt=end), environ=environ, exchange_tz='UTC', asset_db_path=None # We don't need an asset db, we have exchanges ) env.asset_finder = AssetFinderExchange() choose_loader = None # TODO: use the DataPortal for in the algorithm class for this if live: start = pd.Timestamp.utcnow() # TODO: fix the end data. end = start + timedelta(hours=8760) data = DataPortalExchangeLive(exchanges=exchanges, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime( 'today', utc=True)) def fetch_capital_base(exchange, attempt_index=0): """ Fetch the base currency amount required to bootstrap the algorithm against the exchange. The algorithm cannot continue without this value. :param exchange: the targeted exchange :param attempt_index: :return capital_base: the amount of base currency available for trading """ try: log.debug('retrieving capital base in {} to bootstrap ' 'exchange {}'.format(base_currency, exchange_name)) balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: log.warn('could not retrieve balances on {}: {}'.format( exchange.name, e)) sleep(5) return fetch_capital_base(exchange, attempt_index + 1) else: raise ExchangeRequestErrorTooManyAttempts( attempts=attempt_index, error=e) if base_currency in balances: return balances[base_currency] else: raise BaseCurrencyNotFoundError(base_currency=base_currency, exchange=exchange_name) capital_base = 0 for exchange_name in exchanges: exchange = exchanges[exchange_name] capital_base += fetch_capital_base(exchange) sim_params = create_simulation_parameters(start=start, end=end, capital_base=capital_base, emission_rate='minute', data_frequency='minute') # TODO: use the constructor instead sim_params._arena = 'live' algorithm_class = partial(ExchangeTradingAlgorithmLive, exchanges=exchanges, algo_namespace=algo_namespace, live_graph=live_graph) else: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. # I don't think that we should have arbitrary price data bundles # Instead, we should center this data around exchanges. # We still need to support bundles for other misc data, but we # can handle this later. data = DataPortalExchangeBacktest(exchanges=exchanges, asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, last_available_session=end) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) algorithm_class = partial(ExchangeTradingAlgorithmBacktest, exchanges=exchanges) perf = algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def transaction_sim(self, **params): """This is a utility method that asserts expected results for conversion of orders to transactions given a trade history """ trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') asset1 = self.asset_finder.retrieve_asset(1) metadata = make_simple_equity_info([asset1.sid], self.start, self.end) with TempDirectory() as tempdir, \ tmp_trading_env(equities=metadata, load=self.make_load_function()) as env: if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( start=self.start, end=self.end, data_frequency="minute" ) minutes = self.trading_calendar.minutes_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { asset1.sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( self.trading_calendar, self.trading_calendar.sessions_in_range( self.trading_calendar.minute_to_session_label( minutes[0] ), self.trading_calendar.minute_to_session_label( minutes[-1] ) ), tempdir.path, iteritems(assets), ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily" ) days = sim_params.sessions assets = { 1: pd.DataFrame({ "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") BcolzDailyBarWriter(path, self.trading_calendar, days[0], days[-1]).write( assets.items() ) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, equity_daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedSlippage() else: slippage_func = None blotter = Blotter(sim_params.data_frequency, slippage_func) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = PerformanceTracker(sim_params, self.trading_calendar, self.env) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator ** len(order_list) order_id = blotter.order( asset1, order_amount * direction, MarketOrder()) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData( data_portal=data_portal, simulation_dt_func=lambda: tick, data_frequency=sim_params.data_frequency, trading_calendar=self.trading_calendar, restrictions=NoRestrictions(), ) txns, _, closed_orders = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) blotter.prune_orders(closed_orders) for i in range(order_count): order = order_list[i] self.assertEqual(order.asset, asset1) self.assertEqual(order.amount, order_amount * alternator ** i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) cumulative_pos = tracker.position_tracker.positions[asset1] if total_volume == 0: self.assertIsNone(cumulative_pos) else: self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain the asset. oo = blotter.open_orders self.assertNotIn( asset1, oo, "Entry is removed when no open orders" )
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) mode = 'live' if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) if live and exchange is not None: exchange_name = exchange start = pd.Timestamp.utcnow() end = start + timedelta(minutes=1439) portfolio = get_algo_object(algo_name=algo_namespace, key='portfolio_{}'.format(exchange_name), environ=environ) if portfolio is None: portfolio = ExchangePortfolio(start_date=pd.Timestamp.utcnow()) exchange_auth = get_exchange_auth(exchange_name) if exchange_name == 'bitfinex': exchange = Bitfinex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'bittrex': exchange = Bittrex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) else: raise NotImplementedError('exchange not supported: %s' % exchange_name) open_calendar = get_calendar('OPEN') sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) if live and exchange is not None: env = TradingEnvironment(environ=environ, exchange_tz='UTC', asset_db_path=None) env.asset_finder = AssetFinderExchange(exchange) data = DataPortalExchange(exchange=exchange, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime('today', utc=True)) choose_loader = None def fetch_capital_base(attempt_index=0): """ Fetch the base currency amount required to bootstrap the algorithm against the exchange. The algorithm cannot continue without this value. :param attempt_index: :return capital_base: the amount of base currency available for trading """ try: log.debug('retrieving capital base in {} to bootstrap ' 'exchange {}'.format(base_currency, exchange_name)) balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: sleep(5) return fetch_capital_base(attempt_index + 1) else: raise ExchangeRequestErrorTooManyAttempts( attempts=attempt_index, error=e) if base_currency in balances: return balances[base_currency] else: raise BaseCurrencyNotFoundError(base_currency=base_currency, exchange=exchange_name) sim_params = create_simulation_parameters( start=start, end=end, capital_base=fetch_capital_base(), emission_rate='minute', data_frequency='minute') elif bundle is not None: bundles = bundle.split(',') def get_trading_env_and_data(bundles): env = data = None b = 'poloniex' if len(bundles) == 0: return env, data elif len(bundles) == 1: b = bundles[0] bundle_data = load( b, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment( load=partial(load_crypto_market_data, bundle=b, bundle_data=bundle_data, environ=environ), bm_symbol='USDT_BTC', trading_calendar=open_calendar, asset_db_path=connstr, environ=environ, ) first_trading_day = bundle_data.minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, minute_reader=bundle_data.minute_bar_reader, five_minute_reader=bundle_data.five_minute_bar_reader, daily_reader=bundle_data.daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) return env, data def get_loader_for_bundle(b): bundle_data = load( b, environ, bundle_timestamp, ) if b == 'poloniex': return CryptoPricingLoader( bundle_data, data_frequency, CryptoPricing, ) elif b == 'quandl': return USEquityPricingLoader( bundle_data, data_frequency, USEquityPricing, ) raise ValueError("No PipelineLoader registered for bundle %s." % b) loaders = [get_loader_for_bundle(b) for b in bundles] env, data = get_trading_env_and_data(bundles) def choose_loader(column): for loader in loaders: if column in loader.columns: return loader raise ValueError("No PipelineLoader registered for column %s." % column) else: env = TradingEnvironment(environ=environ) choose_loader = None TradingAlgorithmClass = (partial(ExchangeTradingAlgorithm, exchange=exchange, algo_namespace=algo_namespace, live_graph=live_graph) if live and exchange else TradingAlgorithm) perf = TradingAlgorithmClass( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def create_test_catalyst(**config): """ :param config: A configuration object that is a dict with: - sid - an integer, which will be used as the asset ID. - order_count - the number of orders the test algo will place, defaults to 100 - order_amount - the number of shares per order, defaults to 100 - trade_count - the number of trades to simulate, defaults to 101 to ensure all orders are processed. - algorithm - optional parameter providing an algorithm. defaults to :py:class:`catalyst.test.algorithms.TestAlgorithm` - trade_source - optional parameter to specify trades, if present. If not present :py:class:`catalyst.sources.SpecificEquityTrades` is the source, with daily frequency in trades. - slippage: optional parameter that configures the :py:class:`catalyst.gens.tradingsimulation.TransactionSimulator`. Expects an object with a simulate mehod, such as :py:class:`catalyst.gens.tradingsimulation.FixedSlippage`. :py:mod:`catalyst.finance.trading` """ assert isinstance(config, dict) try: sid_list = config['sid_list'] except KeyError: try: sid_list = [config['sid']] except KeyError: raise Exception("simfactory create_test_catalyst() requires " "argument 'sid_list' or 'sid'") concurrent_trades = config.get('concurrent_trades', False) order_count = config.get('order_count', 100) order_amount = config.get('order_amount', 100) trading_calendar = config.get('trading_calendar', get_calendar("NYSE")) # ------------------- # Create the Algo # ------------------- if 'algorithm' in config: test_algo = config['algorithm'] else: test_algo = TestAlgorithm( sid_list[0], order_amount, order_count, sim_params=config.get('sim_params', factory.create_simulation_parameters()), trading_calendar=trading_calendar, slippage=config.get('slippage'), identifiers=sid_list ) # ------------------- # Trade Source # ------------------- if 'skip_data' not in config: if 'trade_source' in config: trade_source = config['trade_source'] else: trade_source = factory.create_daily_trade_source( sid_list, test_algo.sim_params, test_algo.trading_environment, trading_calendar, concurrent=concurrent_trades, ) trades_by_sid = {} for trade in trade_source: if trade.sid not in trades_by_sid: trades_by_sid[trade.sid] = [] trades_by_sid[trade.sid].append(trade) data_portal = create_data_portal_from_trade_history( config['env'].asset_finder, trading_calendar, config['tempdir'], config['sim_params'], trades_by_sid ) test_algo.data_portal = data_portal # ------------------- # Benchmark source # ------------------- test_algo.benchmark_return_source = config.get('benchmark_source', None) # ------------------ # generator/simulator sim = test_algo.get_generator() return sim