def test_unadjusted_spot_price_no_data(self): table = self.bcolz_daily_bar_ctable reader = BcolzDailyBarReader(table) # before with self.assertRaises(NoDataOnDate): reader.spot_price(2, Timestamp('2015-06-08', tz='UTC'), 'close') # after with self.assertRaises(NoDataOnDate): reader.spot_price(4, Timestamp('2015-06-16', tz='UTC'), 'close')
def test_unadjusted_spot_price_no_data(self): table = self.writer.write(self.dest, self.trading_days, self.assets) reader = BcolzDailyBarReader(table) # before with self.assertRaises(NoDataOnDate): reader.spot_price(2, Timestamp('2015-06-08', tz='UTC'), 'close') # after with self.assertRaises(NoDataOnDate): reader.spot_price(4, Timestamp('2015-06-16', tz='UTC'), 'close')
def _check_read_results(self, columns, assets, start_date, end_date): table = self.writer.write(self.dest, self.trading_days, self.assets) reader = BcolzDailyBarReader(table) results = reader.load_raw_arrays(columns, start_date, end_date, assets) dates = self.trading_days_between(start_date, end_date) for column, result in zip(columns, results): assert_array_equal( result, self.writer.expected_values_2d( dates, assets, column.name, ))
def _check_read_results(self, columns, assets, start_date, end_date): table = self.writer.write(self.dest, self.trading_days, self.assets) reader = BcolzDailyBarReader(table) results = reader.load_raw_arrays(columns, start_date, end_date, assets) dates = self.trading_days_between(start_date, end_date) for column, result in zip(columns, results): assert_array_equal( result, self.writer.expected_values_2d( dates, assets, column.name, ) )
def create_data_portal(env, tempdir, sim_params, sids, adjustment_reader=None): if sim_params.data_frequency == "daily": daily_path = write_daily_data(tempdir, sim_params, sids) equity_daily_reader = BcolzDailyBarReader(daily_path) return DataPortal( env, equity_daily_reader=equity_daily_reader, adjustment_reader=adjustment_reader ) else: minutes = env.minutes_for_days_in_range( sim_params.first_open, sim_params.last_close ) minute_path = write_minute_data(env, tempdir, minutes, sids) equity_minute_reader = BcolzMinuteBarReader(minute_path) return DataPortal( env, equity_minute_reader=equity_minute_reader, adjustment_reader=adjustment_reader )
def test_unadjusted_spot_price_empty_value(self): table = self.writer.write(self.dest, self.trading_days, self.assets) reader = BcolzDailyBarReader(table) # A sid, day and corresponding index into which to overwrite a zero. zero_sid = 1 zero_day = Timestamp('2015-06-02', tz='UTC') zero_ix = reader.sid_day_index(zero_sid, zero_day) # Write a zero into the synthetic pricing data at the day and sid, # so that a read should now return -1. # This a little hacky, in lieu of changing the synthetic data set. reader._spot_col('close')[zero_ix] = 0 close = reader.spot_price(zero_sid, zero_day, 'close') self.assertEqual(-1, close)
def create_data_portal(asset_finder, tempdir, sim_params, sids, trading_calendar, adjustment_reader=None): if sim_params.data_frequency == "daily": daily_path = write_daily_data(tempdir, sim_params, sids, trading_calendar) equity_daily_reader = BcolzDailyBarReader(daily_path) return DataPortal( asset_finder, trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, equity_daily_reader=equity_daily_reader, adjustment_reader=adjustment_reader ) else: minutes = trading_calendar.minutes_in_range( sim_params.first_open, sim_params.last_close ) minute_path = write_minute_data(trading_calendar, tempdir, minutes, sids) equity_minute_reader = BcolzMinuteBarReader(minute_path) return DataPortal( asset_finder, trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader, adjustment_reader=adjustment_reader )
def setUpClass(cls): setup_logger(cls) cls.env = trading.TradingEnvironment() cls.sim_params = factory.create_simulation_parameters( start=pd.Timestamp("2006-01-05", tz='UTC'), end=pd.Timestamp("2006-01-06", tz='UTC')) cls.env.write_data( equities_data={ 24: { 'start_date': cls.sim_params.trading_days[0], 'end_date': cls.env.next_trading_day(cls.sim_params.trading_days[-1]) }, 25: { 'start_date': cls.sim_params.trading_days[0], 'end_date': cls.env.next_trading_day(cls.sim_params.trading_days[-1]) } }) cls.tempdir = TempDirectory() assets = { 24: pd.DataFrame({ "open": [50, 50], "high": [50, 50], "low": [50, 50], "close": [50, 50], "volume": [100, 400], "day": [day.value for day in cls.sim_params.trading_days] }), 25: pd.DataFrame({ "open": [50, 50], "high": [50, 50], "low": [50, 50], "close": [50, 50], "volume": [100, 400], "day": [day.value for day in cls.sim_params.trading_days] }) } path = os.path.join(cls.tempdir.path, "tempdata.bcolz") DailyBarWriterFromDataFrames(assets).write(path, cls.sim_params.trading_days, assets) equity_daily_reader = BcolzDailyBarReader(path) cls.data_portal = DataPortal( cls.env, equity_daily_reader=equity_daily_reader, )
def test_unadjusted_spot_price(self): table = self.writer.write(self.dest, self.trading_days, self.assets) reader = BcolzDailyBarReader(table) # At beginning price = reader.spot_price(1, Timestamp('2015-06-01', tz='UTC'), 'close') # Synthetic writes price for date. self.assertEqual(135630.0, price) # Middle price = reader.spot_price(1, Timestamp('2015-06-02', tz='UTC'), 'close') self.assertEqual(135631.0, price) # End price = reader.spot_price(1, Timestamp('2015-06-05', tz='UTC'), 'close') self.assertEqual(135634.0, price) # Another sid at beginning. price = reader.spot_price(2, Timestamp('2015-06-22', tz='UTC'), 'close') self.assertEqual(235651.0, price) # Ensure that volume does not have float adjustment applied. volume = reader.spot_price(1, Timestamp('2015-06-02', tz='UTC'), 'volume') self.assertEqual(145631, volume)
def from_files(cls, pricing_path, adjustments_path): """ Create a loader from a bcolz equity pricing dir and a SQLite adjustments path. Parameters ---------- pricing_path : str Path to a bcolz directory written by a BcolzDailyBarWriter. adjusments_path : str Path to an adjusments db written by a SQLiteAdjustmentWriter. """ return cls(BcolzDailyBarReader(pricing_path), SQLiteAdjustmentReader(adjustments_path))
def create_bar_reader(cls, tempdir): resources = { cls.AAPL: join(TEST_RESOURCE_PATH, 'AAPL.csv'), cls.MSFT: join(TEST_RESOURCE_PATH, 'MSFT.csv'), cls.BRK_A: join(TEST_RESOURCE_PATH, 'BRK-A.csv'), } raw_data = { asset: read_csv(path, parse_dates=['day']).set_index('day') for asset, path in iteritems(resources) } # Add 'price' column as an alias because all kinds of stuff in zipline # depends on it being present. :/ for frame in raw_data.values(): frame['price'] = frame['close'] writer = DailyBarWriterFromCSVs(resources) data_path = tempdir.getpath('testdata.bcolz') table = writer.write(data_path, trading_days, cls.assets) return raw_data, BcolzDailyBarReader(table)
def build_daily_data(cls): path = cls.tempdir.getpath("testdaily.bcolz") dfs = { 1: create_daily_df_for_asset(cls.env, cls.trading_days[0], cls.trading_days[-1]), 2: create_daily_df_for_asset(cls.env, cls.trading_days[0], cls.trading_days[-1]), 3: create_daily_df_for_asset(cls.env, cls.trading_days[0], cls.trading_days[-1]) } daily_writer = DailyBarWriterFromDataFrames(dfs) daily_writer.write(path, cls.trading_days, dfs) return BcolzDailyBarReader(path)
def setUpClass(cls): cls.first_asset_start = Timestamp('2015-04-01', tz='UTC') cls.env = TradingEnvironment() cls.trading_day = day = cls.env.trading_day cls.calendar = date_range('2015', '2015-08', tz='UTC', freq=day) cls.asset_info = make_rotating_asset_info( num_assets=6, first_start=cls.first_asset_start, frequency=day, periods_between_starts=4, asset_lifetime=8, ) cls.last_asset_end = cls.asset_info['end_date'].max() cls.all_assets = cls.asset_info.index cls.env.write_data(equities_df=cls.asset_info) cls.finder = cls.env.asset_finder cls.temp_dir = TempDirectory() cls.temp_dir.create() try: cls.writer = SyntheticDailyBarWriter( asset_info=cls.asset_info[['start_date', 'end_date']], calendar=cls.calendar, ) table = cls.writer.write( cls.temp_dir.getpath('testdata.bcolz'), cls.calendar, cls.all_assets, ) cls.pipeline_loader = USEquityPricingLoader( BcolzDailyBarReader(table), NullAdjustmentReader(), ) except: cls.temp_dir.cleanup() raise
def transaction_sim(self, **params): """ This is a utility method that asserts expected results for conversion of orders to transactions given a trade history""" tempdir = TempDirectory() try: trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') env = TradingEnvironment() sid = 1 if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( data_frequency="minute") minutes = env.market_minute_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( env, env.days_in_range(minutes[0], minutes[-1]), tempdir.path, assets) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env, equity_minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily") days = sim_params.trading_days assets = { 1: pd.DataFrame( { "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") DailyBarWriterFromDataFrames(assets).write(path, days, assets) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env, equity_daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedSlippage() else: slippage_func = None blotter = Blotter(sim_params.data_frequency, self.env.asset_finder, slippage_func) env.write_data( equities_data={ sid: { "start_date": sim_params.trading_days[0], "end_date": sim_params.trading_days[-1] } }) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = PerformanceTracker(sim_params, self.env) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator**len(order_list) order_id = blotter.order( blotter.asset_finder.retrieve_asset(sid), order_amount * direction, MarketOrder()) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData(data_portal, lambda: tick, sim_params.data_frequency) txns, _ = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) for i in range(order_count): order = order_list[i] self.assertEqual(order.sid, sid) self.assertEqual(order.amount, order_amount * alternator**i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) cumulative_pos = tracker.position_tracker.positions[sid] if total_volume == 0: self.assertIsNone(cumulative_pos) else: self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain sid. oo = blotter.open_orders self.assertNotIn(sid, oo, "Entry is removed when no open orders") finally: tempdir.cleanup()
def setUp(self): self.env = env = trading.TradingEnvironment() self.dates = date_range( '2014-01-01', '2014-02-01', freq=trading_day, tz='UTC' ) asset_info = DataFrame.from_records([ { 'sid': 1, 'symbol': 'A', 'start_date': self.dates[10], 'end_date': self.dates[13], 'exchange': 'TEST', }, { 'sid': 2, 'symbol': 'B', 'start_date': self.dates[11], 'end_date': self.dates[14], 'exchange': 'TEST', }, { 'sid': 3, 'symbol': 'C', 'start_date': self.dates[12], 'end_date': self.dates[15], 'exchange': 'TEST', }, ]) self.first_asset_start = min(asset_info.start_date) self.last_asset_end = max(asset_info.end_date) env.write_data(equities_df=asset_info) self.asset_finder = finder = env.asset_finder sids = (1, 2, 3) self.assets = finder.retrieve_all(sids) # View of the baseline data. self.closes = DataFrame( {sid: arange(1, len(self.dates) + 1) * sid for sid in sids}, index=self.dates, dtype=float, ) # Create a data portal holding the data in self.closes data = {} for sid in sids: data[sid] = DataFrame({ "open": self.closes[sid].values, "high": self.closes[sid].values, "low": self.closes[sid].values, "close": self.closes[sid].values, "volume": self.closes[sid].values, "day": [day.value for day in self.dates] }) path = os.path.join(self.tempdir.path, "testdaily.bcolz") DailyBarWriterFromDataFrames(data).write( path, self.dates, data ) daily_bar_reader = BcolzDailyBarReader(path) self.data_portal = DataPortal( self.env, equity_daily_reader=daily_bar_reader, ) # Add a split for 'A' on its second date. self.split_asset = self.assets[0] self.split_date = self.split_asset.start_date + trading_day self.split_ratio = 0.5 self.adjustments = DataFrame.from_records([ { 'sid': self.split_asset.sid, 'value': self.split_ratio, 'kind': MULTIPLY, 'start_date': Timestamp('NaT'), 'end_date': self.split_date, 'apply_date': self.split_date, } ]) # View of the data on/after the split. self.adj_closes = adj_closes = self.closes.copy() adj_closes.ix[:self.split_date, self.split_asset] *= self.split_ratio self.pipeline_loader = DataFrameLoader( column=USEquityPricing.close, baseline=self.closes, adjustments=self.adjustments, )
def create_data_portal_from_trade_history(env, tempdir, sim_params, trades_by_sid): if sim_params.data_frequency == "daily": path = os.path.join(tempdir.path, "testdaily.bcolz") assets = {} for sidint, trades in iteritems(trades_by_sid): opens = [] highs = [] lows = [] closes = [] volumes = [] for trade in trades: opens.append(trade["open_price"]) highs.append(trade["high"]) lows.append(trade["low"]) closes.append(trade["close_price"]) volumes.append(trade["volume"]) assets[sidint] = pd.DataFrame({ "open": np.array(opens), "high": np.array(highs), "low": np.array(lows), "close": np.array(closes), "volume": np.array(volumes), "day": [day.value for day in sim_params.trading_days] }, index=sim_params.trading_days) DailyBarWriterFromDataFrames(assets).write( path, sim_params.trading_days, assets ) equity_daily_reader = BcolzDailyBarReader(path) return DataPortal( env, equity_daily_reader=equity_daily_reader, ) else: minutes = env.minutes_for_days_in_range( sim_params.first_open, sim_params.last_close ) length = len(minutes) assets = {} for sidint, trades in iteritems(trades_by_sid): opens = np.zeros(length) highs = np.zeros(length) lows = np.zeros(length) closes = np.zeros(length) volumes = np.zeros(length) for trade in trades: # put them in the right place idx = minutes.searchsorted(trade.dt) opens[idx] = trade.open_price * 1000 highs[idx] = trade.high * 1000 lows[idx] = trade.low * 1000 closes[idx] = trade.close_price * 1000 volumes[idx] = trade.volume assets[sidint] = pd.DataFrame({ "open": opens, "high": highs, "low": lows, "close": closes, "volume": volumes, "dt": minutes }).set_index("dt") write_bcolz_minute_data( env, env.days_in_range( sim_params.first_open, sim_params.last_close ), tempdir.path, assets ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) return DataPortal( env, equity_minute_reader=equity_minute_reader, )
def test_read_with_adjustments(self): columns = [USEquityPricing.high, USEquityPricing.volume] query_days = self.calendar_days_between(TEST_QUERY_START, TEST_QUERY_STOP) # Our expected results for each day are based on values from the # previous day. shifted_query_days = self.calendar_days_between( TEST_QUERY_START, TEST_QUERY_STOP, shift=-1, ) baseline_reader = BcolzDailyBarReader(self.bcolz_path) adjustment_reader = SQLiteAdjustmentReader(self.db_path) pricing_loader = USEquityPricingLoader( baseline_reader, adjustment_reader, ) highs, volumes = pricing_loader.load_adjusted_array( columns, dates=query_days, assets=Int64Index(arange(1, 7)), mask=ones((len(query_days), 6), dtype=bool), ) expected_baseline_highs = self.bcolz_writer.expected_values_2d( shifted_query_days, self.assets, 'high', ) expected_baseline_volumes = self.bcolz_writer.expected_values_2d( shifted_query_days, self.assets, 'volume', ) # At each point in time, the AdjustedArrays should yield the baseline # with all adjustments up to that date applied. for windowlen in range(1, len(query_days) + 1): for offset, window in enumerate(highs.traverse(windowlen)): baseline = expected_baseline_highs[offset:offset + windowlen] baseline_dates = query_days[offset:offset + windowlen] expected_adjusted_highs = self.apply_adjustments( baseline_dates, self.assets, baseline, # Apply all adjustments. concat([SPLITS, MERGERS, DIVIDENDS_EXPECTED], ignore_index=True), ) assert_allclose(expected_adjusted_highs, window) for offset, window in enumerate(volumes.traverse(windowlen)): baseline = expected_baseline_volumes[offset:offset + windowlen] baseline_dates = query_days[offset:offset + windowlen] # Apply only splits and invert the ratio. adjustments = SPLITS.copy() adjustments.ratio = 1 / adjustments.ratio expected_adjusted_volumes = self.apply_adjustments( baseline_dates, self.assets, baseline, adjustments, ) # FIXME: Make AdjustedArray properly support integral types. assert_array_equal( expected_adjusted_volumes, window.astype(uint32), ) # Verify that we checked up to the longest possible window. with self.assertRaises(WindowLengthTooLong): highs.traverse(windowlen + 1) with self.assertRaises(WindowLengthTooLong): volumes.traverse(windowlen + 1)
def transaction_sim(self, **params): """This is a utility method that asserts expected results for conversion of orders to transactions given a trade history """ trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') asset1 = self.asset_finder.retrieve_asset(1) metadata = make_simple_equity_info([asset1.sid], self.start, self.end) with TempDirectory() as tempdir, \ tmp_trading_env(equities=metadata, load=self.make_load_function()) as env: if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( start=self.start, end=self.end, data_frequency="minute") minutes = self.trading_calendar.minutes_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { asset1.sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( self.trading_calendar, self.trading_calendar.sessions_in_range( self.trading_calendar.minute_to_session_label( minutes[0]), self.trading_calendar.minute_to_session_label( minutes[-1])), tempdir.path, iteritems(assets), ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily") days = sim_params.sessions assets = { 1: pd.DataFrame( { "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") BcolzDailyBarWriter(path, self.trading_calendar, days[0], days[-1]).write(assets.items()) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, equity_daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedBasisPointsSlippage() else: slippage_func = None blotter = SimulationBlotter(sim_params.data_frequency, slippage_func) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = MetricsTracker( trading_calendar=self.trading_calendar, first_session=sim_params.start_session, last_session=sim_params.end_session, capital_base=sim_params.capital_base, emission_rate=sim_params.emission_rate, data_frequency=sim_params.data_frequency, asset_finder=self.asset_finder, metrics=load_metrics_set('none'), ) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator**len(order_list) order_id = blotter.order( asset1, order_amount * direction, MarketOrder(), ) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData( data_portal=data_portal, simulation_dt_func=lambda: tick, data_frequency=sim_params.data_frequency, trading_calendar=self.trading_calendar, restrictions=NoRestrictions(), ) txns, _, closed_orders = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) blotter.prune_orders(closed_orders) for i in range(order_count): order = order_list[i] self.assertEqual(order.asset, asset1) self.assertEqual(order.amount, order_amount * alternator**i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) if total_volume == 0: self.assertRaises(KeyError, lambda: tracker.positions[asset1]) else: cumulative_pos = tracker.positions[asset1] self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain the asset. oo = blotter.open_orders self.assertNotIn(asset1, oo, "Entry is removed when no open orders")
def create_data_portal_from_trade_history(asset_finder, trading_calendar, tempdir, sim_params, trades_by_sid): if sim_params.data_frequency == "daily": path = os.path.join(tempdir.path, "testdaily.bcolz") writer = BcolzDailyBarWriter( path, trading_calendar, sim_params.start_session, sim_params.end_session ) writer.write( trades_by_sid_to_dfs(trades_by_sid, sim_params.sessions), ) equity_daily_reader = BcolzDailyBarReader(path) return DataPortal( asset_finder, trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, equity_daily_reader=equity_daily_reader, ) else: minutes = trading_calendar.minutes_in_range( sim_params.first_open, sim_params.last_close ) length = len(minutes) assets = {} for sidint, trades in iteritems(trades_by_sid): opens = np.zeros(length) highs = np.zeros(length) lows = np.zeros(length) closes = np.zeros(length) volumes = np.zeros(length) for trade in trades: # put them in the right place idx = minutes.searchsorted(trade.dt) opens[idx] = trade.open_price * 1000 highs[idx] = trade.high * 1000 lows[idx] = trade.low * 1000 closes[idx] = trade.close_price * 1000 volumes[idx] = trade.volume assets[sidint] = pd.DataFrame({ "open": opens, "high": highs, "low": lows, "close": closes, "volume": volumes, "dt": minutes }).set_index("dt") write_bcolz_minute_data( trading_calendar, sim_params.sessions, tempdir.path, assets ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) return DataPortal( asset_finder, trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader, )
def test_read_no_adjustments(self): adjustment_reader = NullAdjustmentReader() columns = [USEquityPricing.close, USEquityPricing.volume] query_days = self.calendar_days_between(TEST_QUERY_START, TEST_QUERY_STOP) # Our expected results for each day are based on values from the # previous day. shifted_query_days = self.calendar_days_between( TEST_QUERY_START, TEST_QUERY_STOP, shift=-1, ) adjustments = adjustment_reader.load_adjustments( columns, query_days, self.assets, ) self.assertEqual(adjustments, [{}, {}]) baseline_reader = BcolzDailyBarReader(self.bcolz_path) pricing_loader = USEquityPricingLoader( baseline_reader, adjustment_reader, ) closes, volumes = pricing_loader.load_adjusted_array( columns, dates=query_days, assets=self.assets, mask=ones((len(query_days), len(self.assets)), dtype=bool), ) expected_baseline_closes = self.bcolz_writer.expected_values_2d( shifted_query_days, self.assets, 'close', ) expected_baseline_volumes = self.bcolz_writer.expected_values_2d( shifted_query_days, self.assets, 'volume', ) # AdjustedArrays should yield the same data as the expected baseline. for windowlen in range(1, len(query_days) + 1): for offset, window in enumerate(closes.traverse(windowlen)): assert_array_equal( expected_baseline_closes[offset:offset + windowlen], window, ) for offset, window in enumerate(volumes.traverse(windowlen)): assert_array_equal( expected_baseline_volumes[offset:offset + windowlen], window, ) # Verify that we checked up to the longest possible window. with self.assertRaises(WindowLengthTooLong): closes.traverse(windowlen + 1) with self.assertRaises(WindowLengthTooLong): volumes.traverse(windowlen + 1)