class GetTradeData(object): def __init__(self, pair='XXBTZUSD', timezone='America/New_York', folder='data'): # initiate api api = krakenex.API() self.k = KrakenAPI(api, tier=0, retry=.1) # set pair self.pair = pair self.tz = pytz.timezone(timezone) # set and create folder self.folder = folder os.makedirs(os.path.join(self.folder, pair), exist_ok=True) def download_trade_data(self, since): folder = os.path.join(self.folder, self.pair) # update or new download if since is 0: fs = os.listdir(folder) if len(fs) > 0: fs.sort() last = int(fs[-1].split('.')[0]) else: last = 0 else: last = since # get data while True: try: time.sleep(0.75) fname = os.path.join(folder, '{}.pickle'.format(str(last).zfill(19))) trades, last = self.k.get_recent_trades(pair=self.pair, since=last) # set timezone index = trades.index.tz_localize(pytz.utc).tz_convert(self.tz) trades.index = index # store print('Data Downloader: storing', fname) trades.to_pickle(fname) except AttributeError: # can't tz_localize on empty (last) trades print('Data Downloader: download/update finished!') break def agg_ohlc(self, since, interval=[0]): folder = os.path.join(self.folder, self.pair) if interval == [0]: from config import INTERVALS interval = INTERVALS # fetch files and convert to dataframe fs = os.listdir(folder) fs.sort(reverse=True) if since > 0: fs = [f for f in fs if int(f.split('.')[0]) >= since * 1e9] trades = [] for f in fs: print(f) if f[0] == '.': continue trades.append(pd.read_pickle(os.path.join(folder, f))) trades = pd.concat(trades, axis=0) trades.loc[:, 'cost'] = trades.price * trades.volume # store on disc # fname = self.folder + self.pair + '_trades.pickle' # print('\nData Downloader: storing', fname) # trades.to_pickle(fname) for i in interval: # resample gtrades = trades.resample('{}min'.format(i)) if i == 1: fname = os.path.join(self.folder, 'OHLC', self.pair + '.pickle') else: fname = os.path.join(self.folder, 'OHLC', self.pair + '_{}.pickle'.format(i)) # ohlc, volume ohlc = gtrades.price.ohlc() ohlc.loc[:, 'volume'] = gtrades.volume.sum() ohlc.volume.fillna(0, inplace=True) closes = ohlc.close.fillna(method='pad') ohlc = ohlc.apply(lambda x: x.fillna(closes)) # vwap ohlc.loc[:, 'vwap'] = gtrades.cost.sum() / ohlc.volume ohlc.vwap.fillna(ohlc.close, inplace=True) # count ohlc.loc[:, 'count'] = gtrades.size() # store on disc print('\nData Downloader: storing', fname) ohlc.to_pickle(fname)
class GetTradeData(object): def __init__(self, folder, pair, timezone): # initiate api self.api = krakenex.API() self.k = KrakenAPI(self.api, tier=0, retry=.1) # set pair self.pair = pair self.tz = pytz.timezone(timezone) # set and create folder self.folder = folder os.makedirs(self.folder + pair, exist_ok=True) def download_trade_data(self, since): folder = self.folder + self.pair + '/' # update or new download? if since is 0: fs = os.listdir(folder) if len(fs) > 0: fs.sort() last = int(fs[-1].split('.')[0]) else: last = 0 else: last = since # get data while True: try: fname = folder + '{}.pickle'.format(str(last).zfill(19)) trades, last = self.k.get_recent_trades(pair=self.pair, since=last) # set timezone index = trades.index.tz_localize(pytz.utc).tz_convert(self.tz) trades.index = index # store print('storing', fname) trades.to_pickle(fname) except CallRateLimitError: print('\n this should not happen. please report an issue on ' 'github! thanks. \n') raise except ValueError: print('download/update finished!') break def agg_ohlc(self, interval): folder = self.folder + self.pair + '/' # fetch files and convert to dataframe fs = os.listdir(folder) fs.sort(reverse=True) trades = [] for f in fs: trades.append(pd.read_pickle(folder + f)) trades = pd.concat(trades, axis=0) trades.loc[:, 'cost'] = trades.price * trades.volume # resample gtrades = trades.resample('{}min'.format(interval)) # ohlc, volume ohlc = gtrades.price.ohlc() ohlc.loc[:, 'vol'] = gtrades.volume.sum() ohlc.vol.fillna(0, inplace=True) closes = ohlc.close.fillna(method='pad') ohlc = ohlc.apply(lambda x: x.fillna(closes)) # vwap ohlc.loc[:, 'vwap'] = gtrades.cost.sum() / ohlc.vol ohlc.vwap.fillna(ohlc.close, inplace=True) # count ohlc.loc[:, 'count'] = gtrades.size() # store on disc fname = self.folder + self.pair + '_{}.pickle'.format(interval) print('storing', fname) ohlc.to_pickle(fname)