def main(db_path, debug):
if debug:
logger.setLevel(logging.DEBUG)
db = SqliteDatabase(path=db_path)
end_date = pendulum.now()
step = pendulum.Interval(minutes=1000)
symbols = get_symbols()
logging.info(f'Found {len(symbols)} symbols')
for i, symbol in enumerate(symbols, 1):
# get start date for symbol
# this is either the last entry from the db
# or the trading start date (from json file)
latest_candle_date = db.get_latest_candle_date(symbol)
if latest_candle_date is None:
logging.debug('No previous entries in db. Starting from scratch')
# TODO: handle case when symbol is missing from trading start days
# e.g. symbol is in symbols.json but not in symbols_trading_start_days.json
start_date = symbol_start_date(symbol)
else:
logging.debug('Found previous db entries. Resuming from latest')
start_date = latest_candle_date
logging.info(
f'{i}/{len(symbols)} | {symbol} | Processing from {start_date.to_datetime_string()}'
)
for d1, d2 in date_range(start_date, end_date, step):
logging.debug(f'{d1} -> {d2}')
# returns (max) 1000 candles, one for every minute
candles = get_candles(symbol, d1, d2)
logging.debug(f'Fetched {len(candles)} candles')
if candles:
db.insert_candles(symbol, candles)
# prevent from api rate-limiting
time.sleep(3)
db.close()
def main(db_path, candle_size, debug):
candle_size_int = int(candle_size[:-1])
if debug:
logger.setLevel(logging.DEBUG)
db = SqliteDatabase(path=db_path, candle_size=candle_size)
symbols = get_symbols()
logging.info(f'Found {len(symbols)} symbols')
for i, symbol in enumerate(symbols, 1):
# get start date for symbol
# this is either the last entry from the db
# or the trading start date (from json file)
latest_candle_date = db.get_latest_candle_date(symbol)
if latest_candle_date is None:
logging.debug('No previous entries in db. Starting from scratch')
start_date = 0
logging.info(f'{i}/{len(symbols)} | {symbol} | Processing from beginning')
else:
logging.debug('Found previous db entries. Resuming from latest')
start_date = latest_candle_date
logging.info(f'{i}/{len(symbols)} | {symbol} | Processing from {pd.to_datetime(start_date, unit="ms", utc=True)}')
while True:
# bitfinex is supposed to return 5000 datapoints but often returns fewer
# probably due to not all bars having trades
now = int(pd.Timestamp.utcnow().timestamp() * 1000)
if start_date == 0:
end_date = now
else:
# number of datapoints x candle size x s/min x ms/s x extra factor
end_date = start_date + 5000 * candle_size_int * 60 * 1000 * 100
# request won't work with an end date after the current time
if end_date > now:
end_date = now
fmt_start = pd.to_datetime(start_date, unit='ms', utc=True).strftime('%D %H:%M')
fmt_end = pd.to_datetime(end_date, unit='ms', utc=True).strftime('%D %H:%M')
logging.debug(f'{fmt_start} -> {fmt_end}')
# returns (max) 5000 candles, one for each bar
candles = get_candles(symbol, start_date, end_date, get_earliest=True, candle_size=candle_size)
# import ipdb; ipdb.set_trace()
# df = pd.DataFrame(candles)
# time_diffs = df[0].astype('int').diff().value_counts()
# if len(time_diffs) > 1:
# logging.debug('WARNING: more than one time difference:')
# logging.debug(time_diffs)
# end when we don't see any new data
last_start_date = start_date
start_date = candles[0][0]
if start_date == last_start_date:
logging.debug('Reached latest data, ending')
time.sleep(1)
break
# seems like this modifies the original 'candles' to insert the ticker
logging.debug(f'Fetched {len(candles)} candles')
if candles:
db.insert_candles(symbol, candles)
# prevent from api rate-limiting -- 60 per minute claimed, but seems to be a little slower
time.sleep(1)
db.close()
