def get_historical_candles(self,
symbol,
start_str,
interval=KLINE_INTERVAL_15MINUTE,
end_str=None,
limit=500):
output_data = []
timeframe = bhelp.interval_to_milliseconds(interval)
if type(start_str) == int:
start_ts = start_str
else:
start_ts = bhelp.date_to_milliseconds(start_str)
first_valid_ts = self._get_earliest_valid_timestamp(symbol, interval)
start_ts = max(start_ts, first_valid_ts)
end_ts = None
if end_str:
if type(end_str) == int:
end_ts = end_str
else:
end_ts = bhelp.date_to_milliseconds(end_str)
params = {
"symbol": symbol,
"interval": interval,
"limit": limit,
"startTime": start_ts,
"endTime": end_ts
}
idx = 0
while True:
temp = self.get_candles(**params)
if not len(temp):
break
output_data += temp
params["startTime"] = temp[-1][0]
idx += 1
if len(temp) < limit:
break
params["startTime"] += timeframe
if idx % 3 == 0:
time.sleep(1)
return output_data
def update_csv(filename, pair='BTCUSDT', interval='1m'):
"""Update the save Klines"""
print("Updating CSV")
last_line = next(reverse_readline(filename)).split(
',') # use the reverse generator
time = date_to_milliseconds(last_line[7]) + interval_to_milliseconds(
interval) #get last missed kline open time
klines_diff = client.get_historical_klines(
pair, '1m', time) # fetch difference from api
#print(klines_diff)
for kline in klines_diff: # write new klines to file
print(kline)
kline = convert_format(kline, pair)
write_to_csv([kline], filename)
def get_historical_candles_generator(self,
symbol,
start_str,
interval=KLINE_INTERVAL_15MINUTE,
end_str=None):
limit = 1000
timeframe = bhelp.interval_to_milliseconds(interval)
start_ts = int(start_str) + timeframe
first_valid_ts = self._get_earliest_valid_timestamp(symbol, interval)
start_ts = max(start_ts, first_valid_ts)
end_ts = None
if end_str:
if type(end_str) == int:
end_ts = end_str
else:
end_ts = bhelp.date_to_milliseconds(end_str)
params = {
"symbol": symbol,
"interval": interval,
"limit": limit,
"startTime": start_ts,
"endTime": end_ts
}
while True:
output_data = np.array(self.get_candles(**params))
if len(output_data) == 0 or len(output_data) == 1:
break
if len(output_data) < limit:
output_data = np.delete(output_data, -1, axis=0)
for output in output_data:
yield output
params["startTime"] = int(output_data[-1, 0])
if len(output_data) < limit:
break
params["startTime"] += timeframe
enums.KLINE_INTERVAL_5MINUTE,
enums.KLINE_INTERVAL_15MINUTE,
enums.KLINE_INTERVAL_30MINUTE,
enums.KLINE_INTERVAL_1HOUR,
enums.KLINE_INTERVAL_2HOUR,
enums.KLINE_INTERVAL_4HOUR,
enums.KLINE_INTERVAL_6HOUR,
enums.KLINE_INTERVAL_8HOUR,
enums.KLINE_INTERVAL_12HOUR,
enums.KLINE_INTERVAL_1DAY,
enums.KLINE_INTERVAL_3DAY,
enums.KLINE_INTERVAL_1WEEK,
# enums.KLINE_INTERVAL_1MONTH,
)
kline_intervals = tuple((helpers.interval_to_milliseconds(token), token)
for token in KLINE_INTERVALS)
#msg('kline_intervals:', dumps(kline_intervals))
#sys.exit()
class attrdict(dict):
def __getattr__(self, name):
return self[name]
def __setattr__(self, name, value):
self[name] = value
# Take a sequence of dicts and return a dict of the dicts, where the
# key in the outer dict is the value at that key in each inner dict
def custom_get_new_data(self,
limit: int = 500,
progress_callback=None,
locked=None,
removeFirst=False,
caller=-1) -> List[dict]:
"""
Returns new data from Binance API from timestamp specified, however this one is custom-made.
:param caller: Caller that called this function. Only used for botThread.
:param removeFirst: Boolean whether newest data is removed or not.
:param locked: Signal to emit back to GUI when storing data. Cannot be canceled once here. Used for databases.
:param progress_callback: Signal to emit back to GUI to show progress.
:param limit: Limit per pull.
:return: A list of dictionaries.
"""
# This code below is taken from binance client and slightly refactored.
self.downloadLoop = True
output_data = [] # Initialize our list
timeframe = interval_to_milliseconds(self.interval)
start_ts = total_beginning_timestamp = self.get_latest_timestamp()
end_progress = time.time() * 1000 - total_beginning_timestamp
idx = 0
while True and self.downloadLoop:
tempData = self.binanceClient.get_klines(symbol=self.symbol,
interval=self.interval,
limit=limit,
startTime=start_ts,
endTime=None)
if not len(tempData):
break
output_data += tempData
start_ts = tempData[-1][0]
if progress_callback:
progress = (start_ts -
total_beginning_timestamp) / end_progress * 94
progress_callback.emit(int(progress), "Downloading data...",
caller)
idx += 1
# check if we received less than the required limit and exit the loop
if len(tempData) < limit:
# exit the while loop
break
# increment next call by our timeframe
start_ts += timeframe
# sleep after every 5th call to be kind to the API
if idx % 5 == 0:
time.sleep(1)
if not self.downloadLoop:
progress_callback.emit(-1, "Download canceled.", caller)
return []
if locked:
locked.emit()
if removeFirst: # This should be refactored once data is inserted in the reverse order.
output_data.pop()
progress_callback.emit(95, "Saving data...", caller)
self.insert_data(output_data)
progress_callback.emit(
97, "This may take a while. Dumping data to database...", caller)
if removeFirst: # We don't want current data as it's not the latest data.
self.dump_to_table(self.data[:len(output_data)])
else:
self.dump_to_table(self.data[1:len(output_data)])
progress_callback.emit(100, "Downloaded all new data successfully.",
caller)
self.downloadLoop = False
self.downloadCompleted = True
return self.data
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
# The Python v3.5 or later required
from binance.enums import *
from binance.helpers import interval_to_milliseconds
#-----------------------------------------------------------------------------------------------------------------------
enum_kline_intervals = (KLINE_INTERVAL_1MINUTE,KLINE_INTERVAL_3MINUTE,KLINE_INTERVAL_5MINUTE,KLINE_INTERVAL_15MINUTE, \
KLINE_INTERVAL_30MINUTE,KLINE_INTERVAL_1HOUR,KLINE_INTERVAL_2HOUR,KLINE_INTERVAL_4HOUR, \
KLINE_INTERVAL_6HOUR,KLINE_INTERVAL_8HOUR,KLINE_INTERVAL_12HOUR,KLINE_INTERVAL_1DAY, \
KLINE_INTERVAL_3DAY,KLINE_INTERVAL_1WEEK,KLINE_INTERVAL_1MONTH)
klines = [
v // 1000
for v in [interval_to_milliseconds(v) for v in enum_kline_intervals]
if v is not None
]
#-----------------------------------------------------------------------------------------------------------------------
enum_order_side = (SIDE_BUY, SIDE_SELL)
#-----------------------------------------------------------------------------------------------------------------------
enum_order_status = (ORDER_STATUS_NEW,ORDER_STATUS_PARTIALLY_FILLED,ORDER_STATUS_FILLED,ORDER_STATUS_CANCELED, \
ORDER_STATUS_PENDING_CANCEL,ORDER_STATUS_REJECTED,ORDER_STATUS_EXPIRED)
#-----------------------------------------------------------------------------------------------------------------------
enum_order_type = (ORDER_TYPE_LIMIT,ORDER_TYPE_MARKET,ORDER_TYPE_STOP_LOSS,ORDER_TYPE_STOP_LOSS_LIMIT, \
ORDER_TYPE_TAKE_PROFIT,ORDER_TYPE_TAKE_PROFIT_LIMIT,ORDER_TYPE_LIMIT_MAKER)
#-----------------------------------------------------------------------------------------------------------------------
RATE_LIMITER_REQUESTS = 'REQUEST'
RATE_LIMITER_ORDERS = 'ORDERS'
enum_rate_limiter = (RATE_LIMITER_REQUESTS, RATE_LIMITER_ORDERS)
#-----------------------------------------------------------------------------------------------------------------------
def _future_coin_historical_klines(
self,
symbol,
interval,
start_str,
end_str=None,
limit=500,
):
"""Get Historical Klines from Binance (spot or futures)
See dateparser docs for valid start and end string formats http://dateparser.readthedocs.io/en/latest/
If using offset strings for dates add "UTC" to date string e.g. "now UTC", "11 hours ago UTC"
:param symbol: Name of symbol pair e.g BNBBTC
:type symbol: str
:param interval: Binance Kline interval
:type interval: str
:param start_str: Start date string in UTC format or timestamp in milliseconds
:type start_str: str|int
:param end_str: optional - end date string in UTC format or timestamp in milliseconds (default will fetch everything up to now)
:type end_str: str|int
:param limit: Default 500; max 1000.
:type limit: int
:param limit: Default 500; max 1000.
:type limit: int
:param spot: Historical klines from spot endpoint, otherwise futures
:type spot: bool
:return: list of OHLCV values
"""
# init our list
output_data = []
# setup the max limit
limit = limit
# convert interval to useful value in seconds
timeframe = int(interval_to_milliseconds(interval))
# convert our date strings to milliseconds
if type(start_str) == int:
start_ts = start_str
else:
start_ts = int(date_to_milliseconds(start_str))
# if an end time was passed convert it
end_ts = None
if end_str:
if type(end_str) == int:
end_ts = end_str
else:
end_ts = int(date_to_milliseconds(end_str))
else:
end_ts = start_ts + limit * timeframe
serv_time = self.get_server_time()['serverTime']
end_ts = min(serv_time, end_ts)
end_ts = int(end_ts // 1)
idx = 0
empty_count = 0
while True:
temp_data = self.futures_coin_klines(symbol=symbol,
interval=interval,
limit=limit,
startTime=start_ts,
endTime=end_ts)
# print(f"""start: {datetime.utcfromtimestamp(start_ts/1000).strftime('%Y-%m-%d %H:%M:%S')}\
# end: {datetime.utcfromtimestamp(end_ts/1000).strftime('%Y-%m-%d %H:%M:%S')})
# """)
# handle the case where exactly the limit amount of data was returned last loop
if not len(temp_data):
if not len(output_data):
# if no data, check the next day
start_ts = int(
start_ts +
interval_to_milliseconds(self.KLINE_INTERVAL_1DAY))
else:
# if there was data before, and not anymore, add one to empty count
empty_count += 1
else:
# in case there is data, take the next start as the last end plus 1 interval
start_ts = temp_data[-1][0]
start_ts += timeframe
# reset empty count because we received data
empty_count = 0
# append this loops data to our output data
output_data += temp_data
# set our start timestamp using the last value in the array
idx += 1
# increment next call by our timeframe;
end_ts = start_ts + limit * timeframe
serv_time = self.get_server_time()['serverTime']
# calculation to floor the result and keep int
end_ts = min(serv_time, end_ts)
end_ts = int(end_ts // 1)
# check if we received less than the required limit and exit the loop
if start_ts >= serv_time:
# exit the while loop
break
if '_2' in symbol:
if self._start_ts_over_expiry(start_ts, symbol):
break
# sleep after every 3rd call to be kind to the API
if idx % 3 == 0:
time.sleep(0.5)
if empty_count == 3:
print('3 consecutive empty response')
# if 3 consecutives call dont give data, then stop
break
return output_data