def __init__(self, user: str, settings: dict): self.exchange = 'binance' super().__init__(user, settings) self.max_pos_size_ito_usdt = get_max_pos_size_ito_usdt( settings['symbol'], settings['leverage']) self.cc = init_ccxt(self.exchange, user) self.trade_id = 0
def __init__(self, user: str, settings: dict): super().__init__(user, settings) self.cc = init_ccxt('bybit', user) self.binance_cc = init_ccxt('binance', 'example_user') self.max_leverage = 100
def __init__(self, user: str, settings: dict): super().__init__(user, settings) self.cc = init_ccxt('binance', user) self.trade_id = 0
async def load_trades(exchange: str, user: str, symbol: str, n_days: float) -> pd.DataFrame: def skip_ids(id_, ids_): if id_ in ids_: print('skipping from', id_) while id_ in ids_: id_ -= 1 print(' to', id_) return id_ def load_cache(): cache_filenames = [ f for f in os.listdir(cache_filepath) if '.csv' in f ] if cache_filenames: print('loading cached trades') cache_df = pd.concat( [pd.read_csv(cache_filepath + f) for f in cache_filenames], axis=0) cache_df = cache_df.set_index('trade_id') return cache_df return None if exchange == 'binance': fetch_trades_func = binance_fetch_trades elif exchange == 'bybit': fetch_trades_func = bybit_fetch_trades else: print(exchange, 'not found') return cc = init_ccxt(exchange, user) filepath = make_get_filepath( os.path.join('historical_data', exchange, 'agg_trades_futures', symbol, '')) cache_filepath = make_get_filepath( filepath.replace(symbol, symbol + '_cache')) age_limit = time() - 60 * 60 * 24 * n_days age_limit_millis = age_limit * 1000 print('age_limit', ts_to_date(age_limit)) cache_df = load_cache() trades_df, chunk_lengths = get_downloaded_trades(filepath, age_limit_millis) ids = set() if trades_df is not None: ids.update(trades_df.index) if cache_df is not None: ids.update(cache_df.index) gaps = [] if trades_df is not None and len(trades_df) > 0: # sids = sorted(ids) for i in range(1, len(sids)): if sids[i - 1] + 1 != sids[i]: gaps.append((sids[i - 1], sids[i])) if gaps: print('gaps', gaps) # prev_fetch_ts = time() new_trades = await fetch_trades_func(cc, symbol) k = 0 while True: k += 1 if (break_ := new_trades[0]['timestamp'] <= age_limit_millis) or k % 20 == 0: print('caching trades...') new_tdf = pd.DataFrame(new_trades).set_index('trade_id') cache_filename = f'{cache_filepath}{new_tdf.index[0]}_{new_tdf.index[-1]}.csv' new_tdf.to_csv(cache_filename) new_trades = [new_trades[0]] if break_: break from_id = skip_ids(new_trades[0]['trade_id'] - 1, ids) - 999 # wait at least 0.75 sec between each fetch sleep_for = max(0.0, 0.75 - (time() - prev_fetch_ts)) await asyncio.sleep(sleep_for) prev_fetch_ts = time() new_trades = await fetch_trades_func(cc, symbol, from_id=from_id) + new_trades ids.update([e['trade_id'] for e in new_trades])
def __init__(self, user: str, settings: dict): self.exchange = 'bybit' self.min_notional = 0.0 super().__init__(user, settings) self.cc = init_ccxt(self.exchange, user)
async def load_trades(exchange: str, user: str, symbol: str, n_days: float) -> pd.DataFrame: def skip_ids(id_, ids_): if id_ in ids_: print('skipping from', id_) while id_ in ids_: id_ -= 1 print(' to', from_id) return id_ cc = init_ccxt(exchange, user) try: if exchange == 'binance': fetch_trades_func = binance_fetch_trades elif exchange == 'bybit': fetch_trades_func = bybit_fetch_trades else: print(exchange, 'not found') return filepath = make_get_filepath( f'historical_data/{exchange}/agg_trades_futures/{symbol}/') cache_filepath = make_get_filepath( f'historical_data/{exchange}/agg_trades_futures/{symbol}_cache/') cache_filenames = [ f for f in os.listdir(cache_filepath) if f.endswith('.csv') ] ids = set() if cache_filenames: print('loading cached trades...') cached_trades = pd.concat( [pd.read_csv(cache_filepath + f) for f in cache_filenames], axis=0) cached_trades = cached_trades.set_index('trade_id').sort_index() cached_trades = cached_trades[~cached_trades.index.duplicated()] ids.update(cached_trades.index) else: cached_trades = None age_limit = time() - 60 * 60 * 24 * n_days age_limit_millis = age_limit * 1000 print('age_limit', ts_to_date(age_limit)) chunk_iterator = iter_chunks(exchange, symbol) chunk = next(chunk_iterator) chunks = {} if chunk is None else {int(chunk.index[0]): chunk} if chunk is not None: ids.update(chunk.index) min_id = min(ids) if ids else 0 new_trades = await fetch_trades_func(cc, symbol) cached_ids = set() k = 0 while True: if new_trades[0]['timestamp'] <= age_limit_millis: break from_id = new_trades[0]['trade_id'] - 1 while True: if chunk is None: min_id = 0 break from_id = skip_ids(from_id, ids) if from_id < min_id: chunk = next(chunk_iterator) if chunk is None: min_id = 0 break else: chunks[int(chunk.index[0])] = chunk ids.update(chunk.index) min_id = min(ids) if chunk.timestamp.max() < age_limit_millis: break else: break from_id = skip_ids(from_id, ids) from_id -= 999 new_trades = await fetch_trades_func(cc, symbol, from_id=from_id) + new_trades k += 1 if k % 20 == 0: print('dumping cache') cache_df = pd.DataFrame([ t for t in new_trades if t['trade_id'] not in cached_ids ]).set_index('trade_id') cache_df.to_csv(cache_filepath + str(int(time() * 1000)) + '.csv') cached_ids.update(cache_df.index) new_trades_df = pd.DataFrame(new_trades).set_index('trade_id') trades_updated = pd.concat(list(chunks.values()) + [new_trades_df, cached_trades], axis=0) no_dup = trades_updated[~trades_updated.index.duplicated()] no_dup_sorted = no_dup.sort_index() chunk_size = 100000 chunk_ids = no_dup_sorted.index // chunk_size * chunk_size for g in no_dup_sorted.groupby(chunk_ids): if g[0] not in chunks or len(chunks[g[0]]) != chunk_size: print('dumping chunk', g[0]) g[1].to_csv(f'{filepath}{str(g[0])}.csv') for f in [ f_ for f_ in os.listdir(cache_filepath) if f_.endswith('.csv') ]: os.remove(cache_filepath + f) await cc.close() return no_dup_sorted[no_dup_sorted.timestamp >= age_limit_millis] except KeyboardInterrupt: await cc.close()