def main(db_path, debug): if debug: logger.setLevel(logging.DEBUG) db = SqliteDatabase(path=db_path) end_date = pendulum.now() step = pendulum.Interval(minutes=1000) symbols = get_symbols() logging.info(f'Found {len(symbols)} symbols') for i, symbol in enumerate(symbols, 1): # get start date for symbol # this is either the last entry from the db # or the trading start date (from json file) latest_candle_date = db.get_latest_candle_date(symbol) if latest_candle_date is None: logging.debug('No previous entries in db. Starting from scratch') # TODO: handle case when symbol is missing from trading start days # e.g. symbol is in symbols.json but not in symbols_trading_start_days.json start_date = symbol_start_date(symbol) else: logging.debug('Found previous db entries. Resuming from latest') start_date = latest_candle_date logging.info( f'{i}/{len(symbols)} | {symbol} | Processing from {start_date.to_datetime_string()}' ) for d1, d2 in date_range(start_date, end_date, step): logging.debug(f'{d1} -> {d2}') # returns (max) 1000 candles, one for every minute candles = get_candles(symbol, d1, d2) logging.debug(f'Fetched {len(candles)} candles') if candles: db.insert_candles(symbol, candles) # prevent from api rate-limiting time.sleep(3) db.close()
def main(db_path, candle_size, debug): candle_size_int = int(candle_size[:-1]) if debug: logger.setLevel(logging.DEBUG) db = SqliteDatabase(path=db_path, candle_size=candle_size) symbols = get_symbols() logging.info(f'Found {len(symbols)} symbols') for i, symbol in enumerate(symbols, 1): # get start date for symbol # this is either the last entry from the db # or the trading start date (from json file) latest_candle_date = db.get_latest_candle_date(symbol) if latest_candle_date is None: logging.debug('No previous entries in db. Starting from scratch') start_date = 0 logging.info(f'{i}/{len(symbols)} | {symbol} | Processing from beginning') else: logging.debug('Found previous db entries. Resuming from latest') start_date = latest_candle_date logging.info(f'{i}/{len(symbols)} | {symbol} | Processing from {pd.to_datetime(start_date, unit="ms", utc=True)}') while True: # bitfinex is supposed to return 5000 datapoints but often returns fewer # probably due to not all bars having trades now = int(pd.Timestamp.utcnow().timestamp() * 1000) if start_date == 0: end_date = now else: # number of datapoints x candle size x s/min x ms/s x extra factor end_date = start_date + 5000 * candle_size_int * 60 * 1000 * 100 # request won't work with an end date after the current time if end_date > now: end_date = now fmt_start = pd.to_datetime(start_date, unit='ms', utc=True).strftime('%D %H:%M') fmt_end = pd.to_datetime(end_date, unit='ms', utc=True).strftime('%D %H:%M') logging.debug(f'{fmt_start} -> {fmt_end}') # returns (max) 5000 candles, one for each bar candles = get_candles(symbol, start_date, end_date, get_earliest=True, candle_size=candle_size) # import ipdb; ipdb.set_trace() # df = pd.DataFrame(candles) # time_diffs = df[0].astype('int').diff().value_counts() # if len(time_diffs) > 1: # logging.debug('WARNING: more than one time difference:') # logging.debug(time_diffs) # end when we don't see any new data last_start_date = start_date start_date = candles[0][0] if start_date == last_start_date: logging.debug('Reached latest data, ending') time.sleep(1) break # seems like this modifies the original 'candles' to insert the ticker logging.debug(f'Fetched {len(candles)} candles') if candles: db.insert_candles(symbol, candles) # prevent from api rate-limiting -- 60 per minute claimed, but seems to be a little slower time.sleep(1) db.close()
def main(debug, usemssql, includecandles, includefundings, includetradings, pghost, pgdb, pguser, pgpw): usemssql = False if debug: logger.setLevel(logging.DEBUG) if usemssql: raise ValueError("MSSQL not supported") else: db = SqliteDatabase(pghost, pgdb, pguser, pgpw) print('Using postgres adapter') end_date = pendulum.now() step = pendulum.Duration(minutes=1000) symbols = get_symbols() logging.info(f'Found {len(symbols)} trading symbols') f_symbols = get_f_symbols() logging.info(f'Found {len(f_symbols)} funding symbols') t_symbols = get_t_symbols() logging.info(f'Found {len(t_symbols)} trading symbols') while True: end_date = pendulum.now() if includecandles: for i, symbol in enumerate(symbols, 1): # get start date for symbol # this is either the last entry from the db # or the trading start date (from json file) latest_candle_date = db.get_latest_candle_date(symbol) if latest_candle_date is None: logging.debug( 'No previous entries in db. Starting from scratch') # TODO: handle case when symbol is missing from trading start days # e.g. symbol is in symbols.json but not in symbols_trading_start_days.json start_date = symbol_start_date(symbol) else: logging.debug( 'Found previous db entries. Resuming from latest') start_date = latest_candle_date logging.info( f'{i}/{len(symbols)} | {symbol} | Processing from {start_date.to_datetime_string()}' ) for d1, d2 in date_range(start_date, end_date, step): logging.debug(f'{d1} -> {d2}') # returns (max) 1000 candles, one for every minute candles = get_candles(symbol, d1, d2) logging.debug(f'Fetched {len(candles)} candles') if candles: db.insert_candles(symbol, candles) # prevent from api rate-limiting time.sleep(3) if includefundings: for i, f_symbol in enumerate(f_symbols, 1): latest_funding_date = db.get_latest_funding_date(f_symbol) if latest_funding_date is None: logging.debug( 'No previous entries in db. Starting from scratch') start_date = symbol_start_date(f_symbol) else: logging.debug( 'Found previous db entries. Resuming from latest') start_date = latest_funding_date logging.info( f'{i}/{len(f_symbols)} | {f_symbol} | Processing from {start_date.to_datetime_string()} ' ) prev_start_date = start_date.subtract(days=1) while start_date < end_date and prev_start_date.diff( start_date).in_seconds() > 60: logging.debug(f'Fetching trades from {start_date} ...') f_trades = get_funding_trades(f_symbol, start_date) logging.debug(f'Fetched {len(f_trades)} candles') prev_start_date = start_date if f_trades: db.insert_funding_trades(f_symbol, f_trades) start_date = pendulum.from_timestamp(f_trades[-1][1] / 1000) else: start_date = start_date.add(minutes=10) time.sleep(3) if includetradings: for i, t_symbol in enumerate(t_symbols, 1): latest_trading_date = db.get_latest_trading_date(t_symbol) if latest_trading_date is None: logging.debug( 'No previous entries in db. Starting from scratch') start_date = symbol_start_date(t_symbol) else: logging.debug( 'Found previous db entries. Resuming from latest') start_date = latest_trading_date logging.info( f'{i}/{len(t_symbols)} | {t_symbol} | Processing from {start_date.to_datetime_string()} ' ) while start_date < end_date: logging.debug(f'Fetching trades from {start_date} ...') trades = get_trades(t_symbol, start_date) logging.debug(f'Fetched {len(trades)} trades') if trades: db.insert_trades(t_symbol, trades) start_date = pendulum.from_timestamp(trades[-1][2] / 1000) # VERY EDGE CASE HERE # if there is > 1000 trades during 1s, we can't get other trades than the first 1000 one # because bfx use second based (not ms) timestamp as a search param, and limit is 1000 records # We'll stuck in infinite loop if our next query timestamp does not increase at least by 1s # not much we can't do, but to continue with the next second # since API limit is 1000 per request for a particular timestamp if len(trades) > 1 and abs(trades[-1][2] - trades[0][2]) < 1000: start_date = start_date.add(seconds=1) else: start_date = start_date.add(seconds=10) time.sleep(3) logger.info('Went through all symbols. Start over again!')