def test_historical_candles(self):
symbol = 'BTCUSDT'
start = '2018-09-10 12:00:00'
end = '2018-09-11 12:00:00'
check_dates = BinanceData().candle(symbol,
startTime=start,
endTime=end)
assert check_dates.open_date.iloc[0] == start
assert check_dates.open_date.iloc[-1] == end
limit = 5
check_dates = BinanceData().candle(symbol,
startTime=start,
endTime=end,
limit=5)
assert len(check_dates) == limit
def repair_data(symbol = 'all', verbose=True):
'''Iterate though candles, find missing dates, replace with Binance data.'''
if verbose:
print('Repairing...')
TIME_RES = '1H' # If changed, must change timedelta parameters as well.
# Get data for symbol
if symbol == 'all':
symbols = get_symbols()
else:
symbols = [symbol]
for symbol in symbols:
if verbose:
print(symbol)
print('------------------')
sql = f"select * from candles where symbol = '{symbol}'"
candles = Database().execute(sql)
candles.index = candles.open_date
# Get min, max date
start = candles.open_date.min()
end = candles.open_date.max()
# Build date range
daterange = pd.date_range(start, end, freq=TIME_RES)
# Find holes in data, throw away non-nulls
missing = []
for date in daterange:
if date not in candles.open_date:
missing.append(date)
# Find chunks of continuous dates for Binance API call
chunks = []
chunk = []
for i, current_candle in enumerate(missing, start=1):
if i < len(missing):
next_candle = missing[i]
if next_candle == current_candle + timedelta(hours=1):
chunk.append(current_candle)
else:
chunk.append(current_candle)
chunks.append(chunk)
chunk = []
if i == len(missing)-1:
chunk.append(next_candle)
chunks.append(chunk)
if verbose:
if missing:
print(f'{len(missing)} missing dates found in {len(chunks)} chunks.')
else:
print('No missing dates found!')
if not missing:
print()
continue
# Date conversion functions for mapping
to_binance_ts = lambda x: tb.DateConvert(x).timestamp*1000
to_date = lambda x: tb.DateConvert(x).date
# Get Binance data
add = []
for chunk in chunks:
# Pad to be safe
startTime = tb.DateConvert(min(chunk) - timedelta(hours=10)).date
endTime = tb.DateConvert(max(chunk) + timedelta(hours=10)).date
limit = len(pd.date_range(startTime, endTime, freq=TIME_RES))
add += BinanceData().candle(
symbol = symbol, limit = limit,
startTime = startTime, endTime = endTime
).to_dict('records')
cols = ['open_date', 'open', 'high', 'low', 'close', 'volume',
'close_date', 'quote_asset_volume', 'number_of_trades',
'taker_buy_base_asset_volume', 'taker_buy_quote_asset_volume',
'ignore']
add = pd.DataFrame(add, columns = cols).drop('ignore', axis=1)
add['symbol'] = symbol
add.open_date = add.open_date.map(to_date)
add.close_date = add.close_date.map(to_date)
missing = pd.DataFrame(missing, columns = ['open_date'])
missing.open_date = missing.open_date.map(to_date)
missing['symbol'] = symbol
for col in candles:
if col not in ['open_date', 'symbol']:
missing[col] = None
success = 0
for i, row in missing.iterrows():
missing_date = row.open_date
if add.open_date.isin([missing_date]).any():
missing.iloc[i,:] = add[add.open_date == missing_date].iloc[0,:]
success+=1
if verbose:
print(f'Binance call returned {success} missing dates.')
print(f'Inserting {len(missing)} items into db.')
print()
# return add, missing, endTime, startTime
Database().insert('candles', missing)
def insert_hourly_candles(symbols, startTime=None, endTime=None,
db='autonotrader', debug=False,
verbose=False, datasource=None):
"""
Get candles from the binance API, insert into the database.
- If no startTime or endTime is provided, inserts the most recent
candles.
- If startTime but not endTime is provided, inserts startTime to
most recent candles.
- If endTime is provided but not startTime, inserts the closest candles
to endTime.
Parameters:
-------------
symbols: string | list of strings
Valid symbols for the given exchange. Example: BTCUSDT, ETHBTC
startTime: python datetime object | date string like '%Y-%m-%d %H:%M:%S'
The date at which to begin data collection.
endTime: python datetime object | date string like '%Y-%m-%d %H:%M:%S'
The date at which to end data collection.
db: string
The name of the database to insert to.
debug: boolean
Setting to True will return the DataFrame that was to be inserted into
the database.
verbose: boolean
Setting to True will display a progress bar for data collection and
database inserts.
datasource: initialized exchanges.base.ExchangeData object
"""
if isinstance(symbols, str):
symbols = [symbols]
# From startTime to most recent candle
if startTime and not endTime:
startTime = tb.DateConvert(startTime).datetime
endTime = datetime.utcnow()
# Interval startTime-endTime
elif startTime and endTime:
startTime = tb.DateConvert(startTime).datetime
endTime = tb.DateConvert(endTime).datetime
# Single candle(s) closest to endTime
elif endTime and not startTime:
endTime = tb.DateConvert(endTime).datetime
# TODO generalize
if not datasource:
datasource = BinanceData()
if startTime and endTime:
daterange = pd.date_range(startTime, endTime, freq='1H')
# Calculate total num of chunks and loop iterations for progress bar
total_chunks = len(daterange)//500
if len(daterange) % 500:
total_chunks+=1
total_iterations = total_chunks*len(symbols)
iteration = 0
chunk_num = 1
to_insert = pd.DataFrame()
# API limited to 500 candles, so split date range into chunks if needed
for subrange in tb.chunker(daterange, 500):
sub_startTime = min(subrange)
sub_endTime = max(subrange)
if total_chunks > 1:
sub_endTime+=timedelta(hours=1)
for symbol in symbols:
# if datasource:
candles = datasource.candle(
symbol, startTime=sub_startTime, endTime=sub_endTime
)
to_insert = pd.concat([to_insert, candles])
iteration+=1
if verbose:
tb.progress_bar(
iteration, total_iterations,
f'Getting {symbol}: chunk {chunk_num} of {total_iterations}'
)
chunk_num+=1
# To avoid losing data, insert in chunks if df becomes large
if len(to_insert) >= 4000 and not debug:
if verbose:
tb.progress_bar(
iteration, total_iterations,
'Inserting into db....................'
)
Database(db=db).insert(
'candles', to_insert, auto_format=True, verbose=False
)
to_insert = pd.DataFrame()
if debug:
return to_insert
else:
Database(db=db).insert(
'candles', to_insert, auto_format=True, verbose=False
)
else:
to_insert = pd.DataFrame()
for symbol in symbols:
candles = datasource.candle(
symbol, startTime = startTime, endTime = endTime
)
to_insert = pd.concat([to_insert, candles])
if debug:
return to_insert
else:
Database(db=db).insert(
'candles', to_insert, auto_format=True, verbose=verbose
)
def test_ticker(self):
symbol = 'BTCUSDT'
assert 'price' in BinanceData().ticker(symbol).keys()
def test_all_tickers(self):
all_tickers = BinanceData().all_tickers()
assert not all_tickers.empty
def test_candle_format(self):
symbol = 'BTCUSDT'
assert not BinanceData().candle(symbol).empty
def test_coinlist(self):
symbols = BinanceData().symbols()
assert not symbols.empty, 'Nothing returned from Binance.symbols call'