def setup(self):
log.info('creating bitfinex exchange')
auth_bitfinex = get_exchange_auth('bitfinex')
self.bitfinex = Bitfinex(key=auth_bitfinex['key'],
secret=auth_bitfinex['secret'],
base_currency='usd')
log.info('creating bittrex exchange')
auth_bitfinex = get_exchange_auth('bittrex')
self.bittrex = Bittrex(key=auth_bitfinex['key'],
secret=auth_bitfinex['secret'],
base_currency='usd')
open_calendar = get_calendar('OPEN')
asset_finder = AssetFinderExchange()
self.data_portal_live = DataPortalExchangeLive(
exchanges=dict(bitfinex=self.bitfinex, bittrex=self.bittrex),
asset_finder=asset_finder,
trading_calendar=open_calendar,
first_trading_day=pd.to_datetime('today', utc=True))
self.data_portal_backtest = DataPortalExchangeBacktest(
exchanges=dict(bitfinex=self.bitfinex),
asset_finder=asset_finder,
trading_calendar=open_calendar,
first_trading_day=None # will set dynamically based on assets
)
def __init__(self, exchange):
self.exchange = exchange
self.minutes_per_day = 1440
self.default_ohlc_ratio = 1000000
self._writers = dict()
self._readers = dict()
self.calendar = get_calendar('OPEN')
def get_data_portal(exchange_names):
open_calendar = get_calendar('OPEN')
asset_finder = ExchangeAssetFinder()
data_portal = DataPortalExchangeBacktest(
exchange_names=exchange_names,
asset_finder=asset_finder,
trading_calendar=open_calendar,
first_trading_day=None # will set dynamically based on assets
)
return data_portal
def get_data_portal(exchanges):
open_calendar = get_calendar('OPEN')
asset_finder = ExchangeAssetFinder(exchanges)
exchange_names = [exchange.name for exchange in exchanges]
data_portal = DataPortalExchangeBacktest(
exchange_names=exchange_names,
asset_finder=asset_finder,
trading_calendar=open_calendar,
first_trading_day=None # will set dynamically based on assets
)
return data_portal
def setup(self):
log.info('creating bitfinex exchange')
exchanges = get_exchanges(['bitfinex', 'bittrex', 'poloniex'])
open_calendar = get_calendar('OPEN')
asset_finder = AssetFinderExchange()
self.data_portal_live = DataPortalExchangeLive(
exchanges=exchanges,
asset_finder=asset_finder,
trading_calendar=open_calendar,
first_trading_day=pd.to_datetime('today', utc=True))
self.data_portal_backtest = DataPortalExchangeBacktest(
exchanges=exchanges,
asset_finder=asset_finder,
trading_calendar=open_calendar,
first_trading_day=None # will set dynamically based on assets
)
def setup(self):
log.info('creating bitfinex exchange')
exchanges = get_exchanges(['bitfinex', 'bittrex', 'poloniex'])
open_calendar = get_calendar('OPEN')
asset_finder = ExchangeAssetFinder()
self.data_portal_live = DataPortalExchangeLive(
exchanges=exchanges,
asset_finder=asset_finder,
trading_calendar=open_calendar,
first_trading_day=pd.to_datetime('today', utc=True)
)
self.data_portal_backtest = DataPortalExchangeBacktest(
exchanges=exchanges,
asset_finder=asset_finder,
trading_calendar=open_calendar,
first_trading_day=None # will set dynamically based on assets
)
def __init__(self, *args, **kwargs):
self._data_frequency = kwargs.pop('data_frequency', None)
kwargs.pop('minutes_per_day', None)
kwargs.pop('calendar', None)
end_session = kwargs.pop('end_session', None)
if end_session is not None:
end_session = end_session.floor('1d')
minutes_per_day = 1440 if self._data_frequency == 'minute' else 1
default_ohlc_ratio = kwargs.pop('default_ohlc_ratio', 100000000)
calendar = get_calendar('OPEN')
super(BcolzExchangeBarWriter, self) \
.__init__(*args, **dict(kwargs,
minutes_per_day=minutes_per_day,
default_ohlc_ratio=default_ohlc_ratio,
calendar=calendar,
end_session=end_session
))
def test_daily_data_to_minute_table(self):
exchange_name = 'poloniex'
# Switch between daily and minute for testing
data_frequency = 'daily'
# data_frequency = 'minute'
exchange = get_exchange(exchange_name)
assets = [
exchange.get_asset('eth_btc'),
exchange.get_asset('etc_btc'),
]
start = pd.to_datetime('2017-9-1', utc=True)
end = pd.to_datetime('2017-9-30', utc=True)
# Preparing the bundle folder
root = get_exchange_folder(exchange.name)
path = BUNDLE_NAME_TEMPLATE.format(
root=root,
frequency=data_frequency
)
ensure_directory(path)
exchange_bundle = ExchangeBundle(exchange)
calendar = get_calendar('OPEN')
# We are using a BcolzMinuteBarWriter even though the data is daily
# Each day has a maximum of one bar
# I tried setting the minutes_per_day to 1 will not create
# unnecessary bars
writer = BcolzExchangeBarWriter(
rootdir=path,
data_frequency=data_frequency,
start_session=start,
end_session=end,
write_metadata=True
)
# This will read the daily data in a bundle created by
# the daily writer. It will write to the minute writer which
# we are passing.
# Ingesting a second asset to ensure that multiple chunks
# don't override each other
for asset in assets:
exchange_bundle.ingest_ctable(
asset=asset,
data_frequency=data_frequency,
period='2017',
start_dt=start,
end_dt=end,
writer=writer,
empty_rows_behavior='strip'
)
reader = BcolzExchangeBarReader(rootdir=path,
data_frequency=data_frequency)
# Reading the two assets to ensure that no data was lost
for asset in assets:
sid = asset.sid
daily_values = reader.load_raw_arrays(
fields=['open', 'high', 'low', 'close', 'volume'],
start_dt=start,
end_dt=end,
sids=[sid],
)
print('found {} rows for last ingestion'.format(
len(daily_values[0]))
)
pass
class QuandlBundleTestCase(ZiplineTestCase):
symbols = 'AAPL', 'BRK_A', 'MSFT', 'ZEN'
asset_start = pd.Timestamp('2014-01', tz='utc')
asset_end = pd.Timestamp('2015-01', tz='utc')
bundle = bundles['quandl']
calendar = get_calendar(bundle.calendar_name)
start_date = calendar.first_session
end_date = calendar.last_session
api_key = 'ayylmao'
columns = 'open', 'high', 'low', 'close', 'volume'
def _expected_data(self, asset_finder):
sids = {
symbol: asset_finder.lookup_symbol(
symbol,
self.asset_start,
).sid
for symbol in self.symbols
}
def per_symbol(symbol):
df = pd.read_csv(
test_resource_path('quandl_samples', symbol + '.csv.gz'),
parse_dates=['Date'],
index_col='Date',
usecols=[
'Open',
'High',
'Low',
'Close',
'Volume',
'Date',
'Ex-Dividend',
'Split Ratio',
],
na_values=['NA'],
).rename(
columns={
'Open': 'open',
'High': 'high',
'Low': 'low',
'Close': 'close',
'Volume': 'volume',
'Date': 'date',
'Ex-Dividend': 'ex_dividend',
'Split Ratio': 'split_ratio',
})
df['sid'] = sids[symbol]
return df
all_ = pd.concat(map(per_symbol, self.symbols)).set_index(
'sid',
append=True,
).unstack()
# fancy list comprehension with statements
@list
@apply
def pricing():
for column in self.columns:
vs = all_[column].values
if column == 'volume':
vs = np.nan_to_num(vs)
yield vs
# the first index our written data will appear in the files on disk
start_idx = (
self.calendar.all_sessions.get_loc(self.asset_start, 'ffill') + 1)
# convert an index into the raw dataframe into an index into the
# final data
i = op.add(start_idx)
def expected_dividend_adjustment(idx, symbol):
sid = sids[symbol]
return (1 -
all_.ix[idx,
('ex_dividend', sid)] / all_.ix[idx - 1,
('close', sid)])
adjustments = [
# ohlc
{
# dividends
i(24): [
Float64Multiply(
first_row=0,
last_row=i(24),
first_col=sids['AAPL'],
last_col=sids['AAPL'],
value=expected_dividend_adjustment(24, 'AAPL'),
)
],
i(87): [
Float64Multiply(
first_row=0,
last_row=i(87),
first_col=sids['AAPL'],
last_col=sids['AAPL'],
value=expected_dividend_adjustment(87, 'AAPL'),
)
],
i(150): [
Float64Multiply(
first_row=0,
last_row=i(150),
first_col=sids['AAPL'],
last_col=sids['AAPL'],
value=expected_dividend_adjustment(150, 'AAPL'),
)
],
i(214): [
Float64Multiply(
first_row=0,
last_row=i(214),
first_col=sids['AAPL'],
last_col=sids['AAPL'],
value=expected_dividend_adjustment(214, 'AAPL'),
)
],
i(31): [
Float64Multiply(
first_row=0,
last_row=i(31),
first_col=sids['MSFT'],
last_col=sids['MSFT'],
value=expected_dividend_adjustment(31, 'MSFT'),
)
],
i(90): [
Float64Multiply(
first_row=0,
last_row=i(90),
first_col=sids['MSFT'],
last_col=sids['MSFT'],
value=expected_dividend_adjustment(90, 'MSFT'),
)
],
i(222): [
Float64Multiply(
first_row=0,
last_row=i(222),
first_col=sids['MSFT'],
last_col=sids['MSFT'],
value=expected_dividend_adjustment(222, 'MSFT'),
)
],
# splits
i(108): [
Float64Multiply(
first_row=0,
last_row=i(108),
first_col=sids['AAPL'],
last_col=sids['AAPL'],
value=1.0 / 7.0,
)
],
},
] * (len(self.columns) - 1) + [
# volume
{
i(108): [
Float64Multiply(
first_row=0,
last_row=i(108),
first_col=sids['AAPL'],
last_col=sids['AAPL'],
value=7.0,
)
],
}
]
return pricing, adjustments
def test_bundle(self):
url_map = merge(
{
format_wiki_url(
self.api_key,
symbol,
self.start_date,
self.end_date,
): test_resource_path('quandl_samples', symbol + '.csv.gz')
for symbol in self.symbols
},
{
format_metadata_url(self.api_key, n): test_resource_path(
'quandl_samples',
'metadata-%d.csv.gz' % n,
)
for n in (1, 2)
},
)
catalyst_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': catalyst_root,
'QUANDL_API_KEY': self.api_key,
}
with patch_read_csv(url_map, strict=True):
ingest('quandl', environ=environ)
bundle = load('quandl', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, 'bfill')],
sessions[sessions.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder, )
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
