def temp_pipeline_engine(calendar, sids, random_seed, symbols=None): """ A contextManager that yields a SimplePipelineEngine holding a reference to an AssetFinder generated via tmp_asset_finder. Parameters ---------- calendar : pd.DatetimeIndex Calendar to pass to the constructed PipelineEngine. sids : iterable[int] Sids to use for the temp asset finder. random_seed : int Integer used to seed instances of SeededRandomLoader. symbols : iterable[str], optional Symbols for constructed assets. Forwarded to make_simple_equity_info. """ equity_info = make_simple_equity_info( sids=sids, start_date=calendar[0], end_date=calendar[-1], symbols=symbols, ) loader = make_seeded_random_loader(random_seed, calendar, sids) def get_loader(column): return loader with tmp_asset_finder(equities=equity_info) as finder: yield SimplePipelineEngine(get_loader, calendar, finder)
def temp_pipeline_engine(calendar, sids, random_seed, symbols=None): """ A contextManager that yields a SimplePipelineEngine holding a reference to an AssetFinder generated via tmp_asset_finder. Parameters ---------- calendar : pd.DatetimeIndex Calendar to pass to the constructed PipelineEngine. sids : iterable[int] Sids to use for the temp asset finder. random_seed : int Integer used to seed instances of SeededRandomLoader. symbols : iterable[str], optional Symbols for constructed assets. Forwarded to make_simple_equity_info. """ equity_info = make_simple_equity_info( sids=sids, start_date=calendar[0], end_date=calendar[-1], symbols=symbols, ) loader = make_seeded_random_loader(random_seed, calendar, sids) def get_loader(column): return loader with tmp_asset_finder(equities=equity_info) as finder: yield SimplePipelineEngine(get_loader, calendar, finder)
def __init__(self, url='sqlite:///:memory:', equities=_default_equities, **frames): self._url = url self._eng = None if equities is self._default_equities: equities = make_simple_equity_info( list(map(ord, 'ABC')), pd.Timestamp(0), pd.Timestamp('2015'), ) frames['equities'] = equities self._frames = frames self._eng = None # set in enter and exit
def __init__(self, url='sqlite:///:memory:', equities=_default_equities, **frames): self._url = url self._eng = None if equities is self._default_equities: equities = make_simple_equity_info( list(map(ord, 'ABC')), pd.Timestamp(0), pd.Timestamp('2015'), ) frames['equities'] = equities self._frames = frames self._eng = None # set in enter and exit
def bundle_ingest_create_writers(environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir): self.assertIsNotNone(asset_db_writer) self.assertIsNotNone(minute_bar_writer) self.assertIsNotNone(daily_bar_writer) self.assertIsNotNone(adjustment_writer) equities = make_simple_equity_info( tuple(range(3)), self.START_DATE, self.END_DATE, ) asset_db_writer.write(equities=equities) called[0] = True
def bundle_ingest_create_writers( environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir): self.assertIsNotNone(asset_db_writer) self.assertIsNotNone(minute_bar_writer) self.assertIsNotNone(daily_bar_writer) self.assertIsNotNone(adjustment_writer) equities = make_simple_equity_info( tuple(range(3)), self.START_DATE, self.END_DATE, ) asset_db_writer.write(equities=equities) called[0] = True
def transaction_sim(self, **params): """This is a utility method that asserts expected results for conversion of orders to transactions given a trade history """ trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') asset1 = self.asset_finder.retrieve_asset(1) metadata = make_simple_equity_info([asset1.sid], self.start, self.end) with TempDirectory() as tempdir, \ tmp_trading_env(equities=metadata, load=self.make_load_function()) as env: if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( start=self.start, end=self.end, data_frequency="minute" ) minutes = self.trading_calendar.minutes_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { asset1.sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( self.trading_calendar, self.trading_calendar.sessions_in_range( self.trading_calendar.minute_to_session_label( minutes[0] ), self.trading_calendar.minute_to_session_label( minutes[-1] ) ), tempdir.path, iteritems(assets), ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily" ) days = sim_params.sessions assets = { 1: pd.DataFrame({ "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") BcolzDailyBarWriter(path, self.trading_calendar, days[0], days[-1]).write( assets.items() ) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedSlippage() else: slippage_func = None blotter = Blotter(sim_params.data_frequency, slippage_func) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = PerformanceTracker(sim_params, self.trading_calendar, self.env) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator ** len(order_list) order_id = blotter.order( asset1, order_amount * direction, MarketOrder()) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData( data_portal=data_portal, simulation_dt_func=lambda: tick, data_frequency=sim_params.data_frequency, trading_calendar=self.trading_calendar, restrictions=NoRestrictions(), ) txns, _, closed_orders = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) blotter.prune_orders(closed_orders) for i in range(order_count): order = order_list[i] self.assertEqual(order.asset, asset1) self.assertEqual(order.amount, order_amount * alternator ** i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) cumulative_pos = tracker.position_tracker.positions[asset1] if total_volume == 0: self.assertIsNone(cumulative_pos) else: self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain the asset. oo = blotter.open_orders self.assertNotIn( asset1, oo, "Entry is removed when no open orders" )
def transaction_sim(self, **params): """This is a utility method that asserts expected results for conversion of orders to transactions given a trade history """ trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') asset1 = self.asset_finder.retrieve_asset(1) metadata = make_simple_equity_info([asset1.sid], self.start, self.end) with TempDirectory() as tempdir, \ tmp_trading_env(equities=metadata, load=self.make_load_function()) as env: if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( start=self.start, end=self.end, data_frequency="minute" ) minutes = self.trading_calendar.minutes_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { asset1.sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( self.trading_calendar, self.trading_calendar.sessions_in_range( self.trading_calendar.minute_to_session_label( minutes[0] ), self.trading_calendar.minute_to_session_label( minutes[-1] ) ), tempdir.path, iteritems(assets), ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily" ) days = sim_params.sessions assets = { 1: pd.DataFrame({ "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") BcolzDailyBarWriter(path, self.trading_calendar, days[0], days[-1]).write( assets.items() ) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, equity_daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedSlippage() else: slippage_func = None blotter = Blotter(sim_params.data_frequency, slippage_func) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = PerformanceTracker(sim_params, self.trading_calendar, self.env) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator ** len(order_list) order_id = blotter.order( asset1, order_amount * direction, MarketOrder()) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData( data_portal=data_portal, simulation_dt_func=lambda: tick, data_frequency=sim_params.data_frequency, trading_calendar=self.trading_calendar, restrictions=NoRestrictions(), ) txns, _, closed_orders = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) blotter.prune_orders(closed_orders) for i in range(order_count): order = order_list[i] self.assertEqual(order.asset, asset1) self.assertEqual(order.amount, order_amount * alternator ** i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) cumulative_pos = tracker.position_tracker.positions[asset1] if total_volume == 0: self.assertIsNone(cumulative_pos) else: self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain the asset. oo = blotter.open_orders self.assertNotIn( asset1, oo, "Entry is removed when no open orders" )
def test_ingest(self): calendar = get_calendar('NYSE') sessions = calendar.sessions_in_range(self.START_DATE, self.END_DATE) minutes = calendar.minutes_for_sessions_in_range( self.START_DATE, self.END_DATE, ) sids = tuple(range(3)) equities = make_simple_equity_info( sids, self.START_DATE, self.END_DATE, ) daily_bar_data = make_bar_data(equities, sessions) minute_bar_data = make_bar_data(equities, minutes) first_split_ratio = 0.5 second_split_ratio = 0.1 splits = pd.DataFrame.from_records([ { 'effective_date': str_to_seconds('2014-01-08'), 'ratio': first_split_ratio, 'sid': 0, }, { 'effective_date': str_to_seconds('2014-01-09'), 'ratio': second_split_ratio, 'sid': 1, }, ]) @self.register( 'bundle', calendar_name='NYSE', start_session=self.START_DATE, end_session=self.END_DATE, ) def bundle_ingest(environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir): assert_is(environ, self.environ) asset_db_writer.write(equities=equities) minute_bar_writer.write(minute_bar_data) daily_bar_writer.write(daily_bar_data) adjustment_writer.write(splits=splits) assert_is_instance(calendar, TradingCalendar) assert_is_instance(cache, dataframe_cache) assert_is_instance(show_progress, bool) self.ingest('bundle', environ=self.environ) bundle = self.load('bundle', environ=self.environ) assert_equal(set(bundle.asset_finder.sids), set(sids)) columns = 'open', 'high', 'low', 'close', 'volume' actual = bundle.equity_minute_bar_reader.load_raw_arrays( columns, minutes[0], minutes[-1], sids, ) for actual_column, colname in zip(actual, columns): assert_equal( actual_column, expected_bar_values_2d(minutes, equities, colname), msg=colname, ) actual = bundle.equity_daily_bar_reader.load_raw_arrays( columns, self.START_DATE, self.END_DATE, sids, ) for actual_column, colname in zip(actual, columns): assert_equal( actual_column, expected_bar_values_2d(sessions, equities, colname), msg=colname, ) adjustments_for_cols = bundle.adjustment_reader.load_adjustments( columns, sessions, pd.Index(sids), ) for column, adjustments in zip(columns, adjustments_for_cols[:-1]): # iterate over all the adjustments but `volume` assert_equal( adjustments, { 2: [ Float64Multiply( first_row=0, last_row=2, first_col=0, last_col=0, value=first_split_ratio, ) ], 3: [ Float64Multiply( first_row=0, last_row=3, first_col=1, last_col=1, value=second_split_ratio, ) ], }, msg=column, ) # check the volume, the value should be 1/ratio assert_equal( adjustments_for_cols[-1], { 2: [ Float64Multiply( first_row=0, last_row=2, first_col=0, last_col=0, value=1 / first_split_ratio, ) ], 3: [ Float64Multiply( first_row=0, last_row=3, first_col=1, last_col=1, value=1 / second_split_ratio, ) ], }, msg='volume', )
def test_ingest(self): calendar = get_calendar('NYSE') sessions = calendar.sessions_in_range(self.START_DATE, self.END_DATE) minutes = calendar.minutes_for_sessions_in_range( self.START_DATE, self.END_DATE, ) sids = tuple(range(3)) equities = make_simple_equity_info( sids, self.START_DATE, self.END_DATE, ) daily_bar_data = make_bar_data(equities, sessions) minute_bar_data = make_bar_data(equities, minutes) first_split_ratio = 0.5 second_split_ratio = 0.1 splits = pd.DataFrame.from_records([ { 'effective_date': str_to_seconds('2014-01-08'), 'ratio': first_split_ratio, 'sid': 0, }, { 'effective_date': str_to_seconds('2014-01-09'), 'ratio': second_split_ratio, 'sid': 1, }, ]) @self.register( 'bundle', calendar_name='NYSE', start_session=self.START_DATE, end_session=self.END_DATE, ) def bundle_ingest(environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir): assert_is(environ, self.environ) asset_db_writer.write(equities=equities) minute_bar_writer.write(minute_bar_data) daily_bar_writer.write(daily_bar_data) adjustment_writer.write(splits=splits) assert_is_instance(calendar, TradingCalendar) assert_is_instance(cache, dataframe_cache) assert_is_instance(show_progress, bool) self.ingest('bundle', environ=self.environ) bundle = self.load('bundle', environ=self.environ) assert_equal(set(bundle.asset_finder.sids), set(sids)) columns = 'open', 'high', 'low', 'close', 'volume' actual = bundle.equity_minute_bar_reader.load_raw_arrays( columns, minutes[0], minutes[-1], sids, ) for actual_column, colname in zip(actual, columns): assert_equal( actual_column, expected_bar_values_2d(minutes, equities, colname), msg=colname, ) actual = bundle.equity_daily_bar_reader.load_raw_arrays( columns, self.START_DATE, self.END_DATE, sids, ) for actual_column, colname in zip(actual, columns): assert_equal( actual_column, expected_bar_values_2d(sessions, equities, colname), msg=colname, ) adjustments_for_cols = bundle.adjustment_reader.load_adjustments( columns, sessions, pd.Index(sids), ) for column, adjustments in zip(columns, adjustments_for_cols[:-1]): # iterate over all the adjustments but `volume` assert_equal( adjustments, { 2: [Float64Multiply( first_row=0, last_row=2, first_col=0, last_col=0, value=first_split_ratio, )], 3: [Float64Multiply( first_row=0, last_row=3, first_col=1, last_col=1, value=second_split_ratio, )], }, msg=column, ) # check the volume, the value should be 1/ratio assert_equal( adjustments_for_cols[-1], { 2: [Float64Multiply( first_row=0, last_row=2, first_col=0, last_col=0, value=1 / first_split_ratio, )], 3: [Float64Multiply( first_row=0, last_row=3, first_col=1, last_col=1, value=1 / second_split_ratio, )], }, msg='volume', )