def main(db_path, debug):
if debug:
logger.setLevel(logging.DEBUG)
db = SqliteDatabase(path=db_path)
end_date = pendulum.now()
step = pendulum.Interval(minutes=1000)
symbols = get_symbols()
logging.info(f'Found {len(symbols)} symbols')
for i, symbol in enumerate(symbols, 1):
# get start date for symbol
# this is either the last entry from the db
# or the trading start date (from json file)
latest_candle_date = db.get_latest_candle_date(symbol)
if latest_candle_date is None:
logging.debug('No previous entries in db. Starting from scratch')
# TODO: handle case when symbol is missing from trading start days
# e.g. symbol is in symbols.json but not in symbols_trading_start_days.json
start_date = symbol_start_date(symbol)
else:
logging.debug('Found previous db entries. Resuming from latest')
start_date = latest_candle_date
logging.info(
f'{i}/{len(symbols)} | {symbol} | Processing from {start_date.to_datetime_string()}'
)
for d1, d2 in date_range(start_date, end_date, step):
logging.debug(f'{d1} -> {d2}')
# returns (max) 1000 candles, one for every minute
candles = get_candles(symbol, d1, d2)
logging.debug(f'Fetched {len(candles)} candles')
if candles:
db.insert_candles(symbol, candles)
# prevent from api rate-limiting
time.sleep(3)
db.close()
def main(db_path, candle_size, debug):
candle_size_int = int(candle_size[:-1])
if debug:
logger.setLevel(logging.DEBUG)
db = SqliteDatabase(path=db_path, candle_size=candle_size)
symbols = get_symbols()
logging.info(f'Found {len(symbols)} symbols')
for i, symbol in enumerate(symbols, 1):
# get start date for symbol
# this is either the last entry from the db
# or the trading start date (from json file)
latest_candle_date = db.get_latest_candle_date(symbol)
if latest_candle_date is None:
logging.debug('No previous entries in db. Starting from scratch')
start_date = 0
logging.info(f'{i}/{len(symbols)} | {symbol} | Processing from beginning')
else:
logging.debug('Found previous db entries. Resuming from latest')
start_date = latest_candle_date
logging.info(f'{i}/{len(symbols)} | {symbol} | Processing from {pd.to_datetime(start_date, unit="ms", utc=True)}')
while True:
# bitfinex is supposed to return 5000 datapoints but often returns fewer
# probably due to not all bars having trades
now = int(pd.Timestamp.utcnow().timestamp() * 1000)
if start_date == 0:
end_date = now
else:
# number of datapoints x candle size x s/min x ms/s x extra factor
end_date = start_date + 5000 * candle_size_int * 60 * 1000 * 100
# request won't work with an end date after the current time
if end_date > now:
end_date = now
fmt_start = pd.to_datetime(start_date, unit='ms', utc=True).strftime('%D %H:%M')
fmt_end = pd.to_datetime(end_date, unit='ms', utc=True).strftime('%D %H:%M')
logging.debug(f'{fmt_start} -> {fmt_end}')
# returns (max) 5000 candles, one for each bar
candles = get_candles(symbol, start_date, end_date, get_earliest=True, candle_size=candle_size)
# import ipdb; ipdb.set_trace()
# df = pd.DataFrame(candles)
# time_diffs = df[0].astype('int').diff().value_counts()
# if len(time_diffs) > 1:
# logging.debug('WARNING: more than one time difference:')
# logging.debug(time_diffs)
# end when we don't see any new data
last_start_date = start_date
start_date = candles[0][0]
if start_date == last_start_date:
logging.debug('Reached latest data, ending')
time.sleep(1)
break
# seems like this modifies the original 'candles' to insert the ticker
logging.debug(f'Fetched {len(candles)} candles')
if candles:
db.insert_candles(symbol, candles)
# prevent from api rate-limiting -- 60 per minute claimed, but seems to be a little slower
time.sleep(1)
db.close()
def main(debug, usemssql, includecandles, includefundings, includetradings,
pghost, pgdb, pguser, pgpw):
usemssql = False
if debug:
logger.setLevel(logging.DEBUG)
if usemssql:
raise ValueError("MSSQL not supported")
else:
db = SqliteDatabase(pghost, pgdb, pguser, pgpw)
print('Using postgres adapter')
end_date = pendulum.now()
step = pendulum.Duration(minutes=1000)
symbols = get_symbols()
logging.info(f'Found {len(symbols)} trading symbols')
f_symbols = get_f_symbols()
logging.info(f'Found {len(f_symbols)} funding symbols')
t_symbols = get_t_symbols()
logging.info(f'Found {len(t_symbols)} trading symbols')
while True:
end_date = pendulum.now()
if includecandles:
for i, symbol in enumerate(symbols, 1):
# get start date for symbol
# this is either the last entry from the db
# or the trading start date (from json file)
latest_candle_date = db.get_latest_candle_date(symbol)
if latest_candle_date is None:
logging.debug(
'No previous entries in db. Starting from scratch')
# TODO: handle case when symbol is missing from trading start days
# e.g. symbol is in symbols.json but not in symbols_trading_start_days.json
start_date = symbol_start_date(symbol)
else:
logging.debug(
'Found previous db entries. Resuming from latest')
start_date = latest_candle_date
logging.info(
f'{i}/{len(symbols)} | {symbol} | Processing from {start_date.to_datetime_string()}'
)
for d1, d2 in date_range(start_date, end_date, step):
logging.debug(f'{d1} -> {d2}')
# returns (max) 1000 candles, one for every minute
candles = get_candles(symbol, d1, d2)
logging.debug(f'Fetched {len(candles)} candles')
if candles:
db.insert_candles(symbol, candles)
# prevent from api rate-limiting
time.sleep(3)
if includefundings:
for i, f_symbol in enumerate(f_symbols, 1):
latest_funding_date = db.get_latest_funding_date(f_symbol)
if latest_funding_date is None:
logging.debug(
'No previous entries in db. Starting from scratch')
start_date = symbol_start_date(f_symbol)
else:
logging.debug(
'Found previous db entries. Resuming from latest')
start_date = latest_funding_date
logging.info(
f'{i}/{len(f_symbols)} | {f_symbol} | Processing from {start_date.to_datetime_string()} '
)
prev_start_date = start_date.subtract(days=1)
while start_date < end_date and prev_start_date.diff(
start_date).in_seconds() > 60:
logging.debug(f'Fetching trades from {start_date} ...')
f_trades = get_funding_trades(f_symbol, start_date)
logging.debug(f'Fetched {len(f_trades)} candles')
prev_start_date = start_date
if f_trades:
db.insert_funding_trades(f_symbol, f_trades)
start_date = pendulum.from_timestamp(f_trades[-1][1] /
1000)
else:
start_date = start_date.add(minutes=10)
time.sleep(3)
if includetradings:
for i, t_symbol in enumerate(t_symbols, 1):
latest_trading_date = db.get_latest_trading_date(t_symbol)
if latest_trading_date is None:
logging.debug(
'No previous entries in db. Starting from scratch')
start_date = symbol_start_date(t_symbol)
else:
logging.debug(
'Found previous db entries. Resuming from latest')
start_date = latest_trading_date
logging.info(
f'{i}/{len(t_symbols)} | {t_symbol} | Processing from {start_date.to_datetime_string()} '
)
while start_date < end_date:
logging.debug(f'Fetching trades from {start_date} ...')
trades = get_trades(t_symbol, start_date)
logging.debug(f'Fetched {len(trades)} trades')
if trades:
db.insert_trades(t_symbol, trades)
start_date = pendulum.from_timestamp(trades[-1][2] /
1000)
# VERY EDGE CASE HERE
# if there is > 1000 trades during 1s, we can't get other trades than the first 1000 one
# because bfx use second based (not ms) timestamp as a search param, and limit is 1000 records
# We'll stuck in infinite loop if our next query timestamp does not increase at least by 1s
# not much we can't do, but to continue with the next second
# since API limit is 1000 per request for a particular timestamp
if len(trades) > 1 and abs(trades[-1][2] -
trades[0][2]) < 1000:
start_date = start_date.add(seconds=1)
else:
start_date = start_date.add(seconds=10)
time.sleep(3)
logger.info('Went through all symbols. Start over again!')
