class GetTradeData(object):
def __init__(self,
pair='XXBTZUSD',
timezone='America/New_York',
folder='data'):
# initiate api
api = krakenex.API()
self.k = KrakenAPI(api, tier=0, retry=.1)
# set pair
self.pair = pair
self.tz = pytz.timezone(timezone)
# set and create folder
self.folder = folder
os.makedirs(os.path.join(self.folder, pair), exist_ok=True)
def download_trade_data(self, since):
folder = os.path.join(self.folder, self.pair)
# update or new download
if since is 0:
fs = os.listdir(folder)
if len(fs) > 0:
fs.sort()
last = int(fs[-1].split('.')[0])
else:
last = 0
else:
last = since
# get data
while True:
try:
time.sleep(0.75)
fname = os.path.join(folder,
'{}.pickle'.format(str(last).zfill(19)))
trades, last = self.k.get_recent_trades(pair=self.pair,
since=last)
# set timezone
index = trades.index.tz_localize(pytz.utc).tz_convert(self.tz)
trades.index = index
# store
print('Data Downloader: storing', fname)
trades.to_pickle(fname)
except AttributeError: # can't tz_localize on empty (last) trades
print('Data Downloader: download/update finished!')
break
def agg_ohlc(self, since, interval=[0]):
folder = os.path.join(self.folder, self.pair)
if interval == [0]:
from config import INTERVALS
interval = INTERVALS
# fetch files and convert to dataframe
fs = os.listdir(folder)
fs.sort(reverse=True)
if since > 0:
fs = [f for f in fs if int(f.split('.')[0]) >= since * 1e9]
trades = []
for f in fs:
print(f)
if f[0] == '.':
continue
trades.append(pd.read_pickle(os.path.join(folder, f)))
trades = pd.concat(trades, axis=0)
trades.loc[:, 'cost'] = trades.price * trades.volume
# store on disc
# fname = self.folder + self.pair + '_trades.pickle'
# print('\nData Downloader: storing', fname)
# trades.to_pickle(fname)
for i in interval:
# resample
gtrades = trades.resample('{}min'.format(i))
if i == 1:
fname = os.path.join(self.folder, 'OHLC',
self.pair + '.pickle')
else:
fname = os.path.join(self.folder, 'OHLC',
self.pair + '_{}.pickle'.format(i))
# ohlc, volume
ohlc = gtrades.price.ohlc()
ohlc.loc[:, 'volume'] = gtrades.volume.sum()
ohlc.volume.fillna(0, inplace=True)
closes = ohlc.close.fillna(method='pad')
ohlc = ohlc.apply(lambda x: x.fillna(closes))
# vwap
ohlc.loc[:, 'vwap'] = gtrades.cost.sum() / ohlc.volume
ohlc.vwap.fillna(ohlc.close, inplace=True)
# count
ohlc.loc[:, 'count'] = gtrades.size()
# store on disc
print('\nData Downloader: storing', fname)
ohlc.to_pickle(fname)
class GetTradeData(object):
def __init__(self, folder, pair, timezone):
# initiate api
self.api = krakenex.API()
self.k = KrakenAPI(self.api, tier=0, retry=.1)
# set pair
self.pair = pair
self.tz = pytz.timezone(timezone)
# set and create folder
self.folder = folder
os.makedirs(self.folder + pair, exist_ok=True)
def download_trade_data(self, since):
folder = self.folder + self.pair + '/'
# update or new download?
if since is 0:
fs = os.listdir(folder)
if len(fs) > 0:
fs.sort()
last = int(fs[-1].split('.')[0])
else:
last = 0
else:
last = since
# get data
while True:
try:
fname = folder + '{}.pickle'.format(str(last).zfill(19))
trades, last = self.k.get_recent_trades(pair=self.pair,
since=last)
# set timezone
index = trades.index.tz_localize(pytz.utc).tz_convert(self.tz)
trades.index = index
# store
print('storing', fname)
trades.to_pickle(fname)
except CallRateLimitError:
print('\n this should not happen. please report an issue on '
'github! thanks. \n')
raise
except ValueError:
print('download/update finished!')
break
def agg_ohlc(self, interval):
folder = self.folder + self.pair + '/'
# fetch files and convert to dataframe
fs = os.listdir(folder)
fs.sort(reverse=True)
trades = []
for f in fs:
trades.append(pd.read_pickle(folder + f))
trades = pd.concat(trades, axis=0)
trades.loc[:, 'cost'] = trades.price * trades.volume
# resample
gtrades = trades.resample('{}min'.format(interval))
# ohlc, volume
ohlc = gtrades.price.ohlc()
ohlc.loc[:, 'vol'] = gtrades.volume.sum()
ohlc.vol.fillna(0, inplace=True)
closes = ohlc.close.fillna(method='pad')
ohlc = ohlc.apply(lambda x: x.fillna(closes))
# vwap
ohlc.loc[:, 'vwap'] = gtrades.cost.sum() / ohlc.vol
ohlc.vwap.fillna(ohlc.close, inplace=True)
# count
ohlc.loc[:, 'count'] = gtrades.size()
# store on disc
fname = self.folder + self.pair + '_{}.pickle'.format(interval)
print('storing', fname)
ohlc.to_pickle(fname)
