def setup(self): log.info('creating bitfinex exchange') auth_bitfinex = get_exchange_auth('bitfinex') self.bitfinex = Bitfinex(key=auth_bitfinex['key'], secret=auth_bitfinex['secret'], base_currency='usd') log.info('creating bittrex exchange') auth_bitfinex = get_exchange_auth('bittrex') self.bittrex = Bittrex(key=auth_bitfinex['key'], secret=auth_bitfinex['secret'], base_currency='usd') open_calendar = get_calendar('OPEN') asset_finder = AssetFinderExchange() self.data_portal_live = DataPortalExchangeLive( exchanges=dict(bitfinex=self.bitfinex, bittrex=self.bittrex), asset_finder=asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime('today', utc=True)) self.data_portal_backtest = DataPortalExchangeBacktest( exchanges=dict(bitfinex=self.bitfinex), asset_finder=asset_finder, trading_calendar=open_calendar, first_trading_day=None # will set dynamically based on assets )
def __init__(self, exchange): self.exchange = exchange self.minutes_per_day = 1440 self.default_ohlc_ratio = 1000000 self._writers = dict() self._readers = dict() self.calendar = get_calendar('OPEN')
def get_data_portal(exchange_names): open_calendar = get_calendar('OPEN') asset_finder = ExchangeAssetFinder() data_portal = DataPortalExchangeBacktest( exchange_names=exchange_names, asset_finder=asset_finder, trading_calendar=open_calendar, first_trading_day=None # will set dynamically based on assets ) return data_portal
def get_data_portal(exchanges): open_calendar = get_calendar('OPEN') asset_finder = ExchangeAssetFinder(exchanges) exchange_names = [exchange.name for exchange in exchanges] data_portal = DataPortalExchangeBacktest( exchange_names=exchange_names, asset_finder=asset_finder, trading_calendar=open_calendar, first_trading_day=None # will set dynamically based on assets ) return data_portal
def setup(self): log.info('creating bitfinex exchange') exchanges = get_exchanges(['bitfinex', 'bittrex', 'poloniex']) open_calendar = get_calendar('OPEN') asset_finder = AssetFinderExchange() self.data_portal_live = DataPortalExchangeLive( exchanges=exchanges, asset_finder=asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime('today', utc=True)) self.data_portal_backtest = DataPortalExchangeBacktest( exchanges=exchanges, asset_finder=asset_finder, trading_calendar=open_calendar, first_trading_day=None # will set dynamically based on assets )
def setup(self): log.info('creating bitfinex exchange') exchanges = get_exchanges(['bitfinex', 'bittrex', 'poloniex']) open_calendar = get_calendar('OPEN') asset_finder = ExchangeAssetFinder() self.data_portal_live = DataPortalExchangeLive( exchanges=exchanges, asset_finder=asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime('today', utc=True) ) self.data_portal_backtest = DataPortalExchangeBacktest( exchanges=exchanges, asset_finder=asset_finder, trading_calendar=open_calendar, first_trading_day=None # will set dynamically based on assets )
def __init__(self, *args, **kwargs): self._data_frequency = kwargs.pop('data_frequency', None) kwargs.pop('minutes_per_day', None) kwargs.pop('calendar', None) end_session = kwargs.pop('end_session', None) if end_session is not None: end_session = end_session.floor('1d') minutes_per_day = 1440 if self._data_frequency == 'minute' else 1 default_ohlc_ratio = kwargs.pop('default_ohlc_ratio', 100000000) calendar = get_calendar('OPEN') super(BcolzExchangeBarWriter, self) \ .__init__(*args, **dict(kwargs, minutes_per_day=minutes_per_day, default_ohlc_ratio=default_ohlc_ratio, calendar=calendar, end_session=end_session ))
def test_daily_data_to_minute_table(self): exchange_name = 'poloniex' # Switch between daily and minute for testing data_frequency = 'daily' # data_frequency = 'minute' exchange = get_exchange(exchange_name) assets = [ exchange.get_asset('eth_btc'), exchange.get_asset('etc_btc'), ] start = pd.to_datetime('2017-9-1', utc=True) end = pd.to_datetime('2017-9-30', utc=True) # Preparing the bundle folder root = get_exchange_folder(exchange.name) path = BUNDLE_NAME_TEMPLATE.format( root=root, frequency=data_frequency ) ensure_directory(path) exchange_bundle = ExchangeBundle(exchange) calendar = get_calendar('OPEN') # We are using a BcolzMinuteBarWriter even though the data is daily # Each day has a maximum of one bar # I tried setting the minutes_per_day to 1 will not create # unnecessary bars writer = BcolzExchangeBarWriter( rootdir=path, data_frequency=data_frequency, start_session=start, end_session=end, write_metadata=True ) # This will read the daily data in a bundle created by # the daily writer. It will write to the minute writer which # we are passing. # Ingesting a second asset to ensure that multiple chunks # don't override each other for asset in assets: exchange_bundle.ingest_ctable( asset=asset, data_frequency=data_frequency, period='2017', start_dt=start, end_dt=end, writer=writer, empty_rows_behavior='strip' ) reader = BcolzExchangeBarReader(rootdir=path, data_frequency=data_frequency) # Reading the two assets to ensure that no data was lost for asset in assets: sid = asset.sid daily_values = reader.load_raw_arrays( fields=['open', 'high', 'low', 'close', 'volume'], start_dt=start, end_dt=end, sids=[sid], ) print('found {} rows for last ingestion'.format( len(daily_values[0])) ) pass
class QuandlBundleTestCase(ZiplineTestCase): symbols = 'AAPL', 'BRK_A', 'MSFT', 'ZEN' asset_start = pd.Timestamp('2014-01', tz='utc') asset_end = pd.Timestamp('2015-01', tz='utc') bundle = bundles['quandl'] calendar = get_calendar(bundle.calendar_name) start_date = calendar.first_session end_date = calendar.last_session api_key = 'ayylmao' columns = 'open', 'high', 'low', 'close', 'volume' def _expected_data(self, asset_finder): sids = { symbol: asset_finder.lookup_symbol( symbol, self.asset_start, ).sid for symbol in self.symbols } def per_symbol(symbol): df = pd.read_csv( test_resource_path('quandl_samples', symbol + '.csv.gz'), parse_dates=['Date'], index_col='Date', usecols=[ 'Open', 'High', 'Low', 'Close', 'Volume', 'Date', 'Ex-Dividend', 'Split Ratio', ], na_values=['NA'], ).rename( columns={ 'Open': 'open', 'High': 'high', 'Low': 'low', 'Close': 'close', 'Volume': 'volume', 'Date': 'date', 'Ex-Dividend': 'ex_dividend', 'Split Ratio': 'split_ratio', }) df['sid'] = sids[symbol] return df all_ = pd.concat(map(per_symbol, self.symbols)).set_index( 'sid', append=True, ).unstack() # fancy list comprehension with statements @list @apply def pricing(): for column in self.columns: vs = all_[column].values if column == 'volume': vs = np.nan_to_num(vs) yield vs # the first index our written data will appear in the files on disk start_idx = ( self.calendar.all_sessions.get_loc(self.asset_start, 'ffill') + 1) # convert an index into the raw dataframe into an index into the # final data i = op.add(start_idx) def expected_dividend_adjustment(idx, symbol): sid = sids[symbol] return (1 - all_.ix[idx, ('ex_dividend', sid)] / all_.ix[idx - 1, ('close', sid)]) adjustments = [ # ohlc { # dividends i(24): [ Float64Multiply( first_row=0, last_row=i(24), first_col=sids['AAPL'], last_col=sids['AAPL'], value=expected_dividend_adjustment(24, 'AAPL'), ) ], i(87): [ Float64Multiply( first_row=0, last_row=i(87), first_col=sids['AAPL'], last_col=sids['AAPL'], value=expected_dividend_adjustment(87, 'AAPL'), ) ], i(150): [ Float64Multiply( first_row=0, last_row=i(150), first_col=sids['AAPL'], last_col=sids['AAPL'], value=expected_dividend_adjustment(150, 'AAPL'), ) ], i(214): [ Float64Multiply( first_row=0, last_row=i(214), first_col=sids['AAPL'], last_col=sids['AAPL'], value=expected_dividend_adjustment(214, 'AAPL'), ) ], i(31): [ Float64Multiply( first_row=0, last_row=i(31), first_col=sids['MSFT'], last_col=sids['MSFT'], value=expected_dividend_adjustment(31, 'MSFT'), ) ], i(90): [ Float64Multiply( first_row=0, last_row=i(90), first_col=sids['MSFT'], last_col=sids['MSFT'], value=expected_dividend_adjustment(90, 'MSFT'), ) ], i(222): [ Float64Multiply( first_row=0, last_row=i(222), first_col=sids['MSFT'], last_col=sids['MSFT'], value=expected_dividend_adjustment(222, 'MSFT'), ) ], # splits i(108): [ Float64Multiply( first_row=0, last_row=i(108), first_col=sids['AAPL'], last_col=sids['AAPL'], value=1.0 / 7.0, ) ], }, ] * (len(self.columns) - 1) + [ # volume { i(108): [ Float64Multiply( first_row=0, last_row=i(108), first_col=sids['AAPL'], last_col=sids['AAPL'], value=7.0, ) ], } ] return pricing, adjustments def test_bundle(self): url_map = merge( { format_wiki_url( self.api_key, symbol, self.start_date, self.end_date, ): test_resource_path('quandl_samples', symbol + '.csv.gz') for symbol in self.symbols }, { format_metadata_url(self.api_key, n): test_resource_path( 'quandl_samples', 'metadata-%d.csv.gz' % n, ) for n in (1, 2) }, ) catalyst_root = self.enter_instance_context(tmp_dir()).path environ = { 'ZIPLINE_ROOT': catalyst_root, 'QUANDL_API_KEY': self.api_key, } with patch_read_csv(url_map, strict=True): ingest('quandl', environ=environ) bundle = load('quandl', environ=environ) sids = 0, 1, 2, 3 assert_equal(set(bundle.asset_finder.sids), set(sids)) for equity in bundle.asset_finder.retrieve_all(sids): assert_equal(equity.start_date, self.asset_start, msg=equity) assert_equal(equity.end_date, self.asset_end, msg=equity) sessions = self.calendar.all_sessions actual = bundle.equity_daily_bar_reader.load_raw_arrays( self.columns, sessions[sessions.get_loc(self.asset_start, 'bfill')], sessions[sessions.get_loc(self.asset_end, 'ffill')], sids, ) expected_pricing, expected_adjustments = self._expected_data( bundle.asset_finder, ) assert_equal(actual, expected_pricing, array_decimal=2) adjustments_for_cols = bundle.adjustment_reader.load_adjustments( self.columns, sessions, pd.Index(sids), ) for column, adjustments, expected in zip(self.columns, adjustments_for_cols, expected_adjustments): assert_equal( adjustments, expected, msg=column, )