def __find_start_trade_id(self, pair, start_unix):
"""Find starting trade ID given start UNIX.
Args:
pair (str): Currency pair.
start_unix (int): Start UNIX to map to a start trade ID (inclusive).
Returns:
Earliest trade ID after time `start_unix`.
"""
# check if no pair trades exist before start_unix
r = self.__get_slice(pair, 1 + self.__MAX_LIMIT)
oldest_t = timeutil.iso_to_unix(r[-1]['time'])
if oldest_t > start_unix:
print('GDAX\t| No trades exist for {} before {}'.format(
pair, start_unix))
return 1
# binary search start trade ID
start, end = 1, self.__find_last_trade_id(pair)
while start <= end:
mid = (start + end) // 2
r = self.__get_slice(pair, mid + 1)
newest_t = timeutil.iso_to_unix(r[0]['time'])
oldest_t = timeutil.iso_to_unix(r[-1]['time'])
if start_unix < oldest_t:
end = mid
elif start_unix > newest_t:
start = mid
else:
# find trade start ID in the returned list
for row in r:
current_t = timeutil.iso_to_unix(row['time'])
if start_unix > current_t:
return row['trade_id'] + 1
def __find_start_trade_id(self, pair, start_unix):
"""Find starting trade ID given start UNIX.
Args:
pair (str): Currency pair.
start_unix (int): Start UNIX to map to a start trade ID (inclusive).
Returns:
Earliest trade ID after time `start_unix`.
"""
# check if no pair trades exist before start_unix
r = self.__get_slice(pair, 0)
oldest_t = timeutil.iso_to_unix(r[0]['timestamp'])
if oldest_t > start_unix:
print('Bitmex\t| No trades exist for {} before {}'.format(
pair, start_unix))
return 0
# incrementally search start trade ID
current_id = 0
increment = 10000000
while True:
current_id += increment
# old -> new
r = self.__get_slice(pair, current_id)
print(current_id)
if len(r) == 0:
current_id -= increment
increment = increment // 2
continue
oldest_t = timeutil.iso_to_unix(r[0]['timestamp'])
newest_t = timeutil.iso_to_unix(r[-1]['timestamp'])
if start_unix > oldest_t and start_unix < newest_t:
# answer is in this page
for row in r:
if timeutil.iso_to_unix(row['timestamp']) >= start_unix:
return current_id
current_id += 1
elif start_unix > newest_t:
pass
elif start_unix < oldest_t:
current_id -= increment
increment = increment // 2
elif start_unix > oldest_t and len(r) < self.__MAX_LIMIT:
# last page
for row in r:
if timeutil.iso_to_unix(row['timestamp']) >= start_unix:
return current_id
current_id += 1
return current_id # starting at this id will return empty list
def __get_slice(self, pair, start):
"""Makes a call to the API that fetches trade data.
Example call:
https://poloniex.com/public?command=returnTradeHistory¤cyPair=BTC_NXT&start=1410158341&end=1410499372
Args:
pair (str): Currency pair.
start (int): UNIX of oldest trade data (inclusive).
Returns:
Trade data from time `start`, up to time `end`, in the order of
increasing UNIX.
"""
params = {'currencyPair': pair,
'start': start,
'end': start + self.__MAX_RANGE}
# new -> old
r = self.__get('/public?command=returnTradeHistory', params)
if len(r) >= self.__MAX_LIMIT:
oldest_date = r[-1]['date']
oldest_date = timeutil.iso_to_unix(oldest_date)
r.extend(self.__get_slice(pair, start, oldest_date))
return r
def __find_last_trade_time(self, pair):
"""Find the latest time of trade for a given pair.
Args:
pair (str): Currency pair.
Returns:
UNIX of last trade for `pair`.
"""
params = {'currencyPair': pair}
# new -> old
r = self.__get('/public?command=returnTradeHistory', params)
return timeutil.iso_to_unix(r[0]['date'])
def download_data(self, pair, start, end):
"""Download trade data and store as .csv file.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data to download.
end (int): End UNIX of trade data to download.
"""
dataio = DataIO(savedir=self._savedir, fieldnames=self.FIELDNAMES)
if dataio.csv_check(pair):
last_row = dataio.csv_get_last(pair)
newest_id = int(last_row['trade_id']) + 1
newest_t = int(last_row['time'])
else:
newest_id = self.__find_start_trade_id(pair, start)
newest_t = 0
last_trade_id = self.__find_last_trade_id(pair)
while newest_t < end:
# new -> old
r = self.__get_slice(pair, newest_id + self.__MAX_LIMIT)
# break condition
to_break = False
# old -> new, add unix timestamp
new_r = []
for row in reversed(r):
if row['trade_id'] > newest_id:
row['date'] = row['time']
row['time'] = timeutil.iso_to_unix(row['time'])
new_r.append(row)
if row['trade_id'] == last_trade_id:
to_break = True
# save to file
dataio.csv_append(pair, new_r)
# break condition
if to_break:
break
# prepare next iteration
newest_id = new_r[-1]['trade_id']
newest_t = new_r[-1]['time']
print('GDAX\t| {} : {}'.format(timeutil.unix_to_iso(newest_t),
pair))
print('GDAX\t| Download complete : {}'.format(pair))
def download_data(self, pair, start, end):
"""Download trade data and store as .csv file.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data to download.
end (int): End UNIX of trade data to download.
"""
dataio = DataIO(savedir=self._savedir, fieldnames=self.FIELDNAMES)
if dataio.csv_check(pair):
last_row = dataio.csv_get_last(pair)
newest_id = int(last_row['trade_id']) + 1
newest_t = int(last_row['time'])
else:
newest_id = self.__find_start_trade_id(pair, start)
newest_t = 0
while newest_t < end:
# new -> old
r = self.__get_slice(pair, newest_id)
# old -> new, add unix timestamp
new_r = []
for i, row in enumerate(r):
row['time'] = timeutil.iso_to_unix(row['timestamp'])
row['date'] = row['timestamp']
row['trade_id'] = newest_id + i
row['side'] = row['side'].lower()
row.pop('timestamp', None)
row.pop('symbol', None)
new_r.append(row)
# save to file
dataio.csv_append(pair, new_r)
# break condition
if len(r) < self.__MAX_LIMIT:
break
# prepare next iteration
newest_id = new_r[-1]['trade_id'] + 1
newest_t = new_r[-1]['time']
print('Bitmex\t| {} : {}'.format(
timeutil.unix_to_iso(newest_t), pair))
print('Bitmex\t| Download complete : {}'.format(pair))
def download_data(self, pair, start, end):
"""Download trade data and store as .csv file.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data to download.
end (int): End UNIX of trade data to download.
"""
dataio = DataIO(savedir=self._savedir, fieldnames=self.FIELDNAMES)
last_row = None
if dataio.csv_check(pair):
last_row = dataio.csv_get_last(pair)
newest_t = int(last_row['time'])
else:
newest_t = self.__find_start_trade_time(pair, start) - 1
# break condition
last_trade_time = self.__find_last_trade_time(pair)
while newest_t < end:
# new -> old
r = self.__get_slice(pair, newest_t)
# old -> new; remove duplicate data by trade ID
new_r = []
for row in reversed(r):
if last_row is not None:
if int(last_row['tradeID']) >= row['tradeID']:
continue # remove duplicates
last_row = row
row['time'] = timeutil.iso_to_unix(row['date'])
new_r.append(row)
if newest_t > last_trade_time:
break
# save to file
dataio.csv_append(pair, new_r)
# prepare next iteration
newest_t += self.__MAX_RANGE
print('Poloniex| {} : {}'.format(
timeutil.unix_to_iso(newest_t), pair))
print('Poloniex| Download complete : {}'.format(pair))
def __find_start_trade_time(self, pair, start_unix):
"""Find a valid start UNIX, given arbitrary start UNIX.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data check if valid.
Returns:
A valid start UNIX, either `start_unix` or UNIX of first trade
ever made for `pair`.
"""
# check if no pair trades exist before start_unix
r = self.__get_slice(pair, start_unix)
while len(r) == 0:
start_unix += 1000000
r = self.__get_slice(pair, start_unix)
oldest_t = timeutil.iso_to_unix(r[-1]['date'])
return oldest_t
import timeutil
if __name__ == '__main__':
unix = 1500000000
# unix to ...
iso = timeutil.unix_to_iso(unix)
utc_date = timeutil.unix_to_utc_date(unix)
local_date = timeutil.unix_to_local_date(unix)
print('unix conversion:', iso)
print('unix conversion:', utc_date)
print('unix conversion:', local_date)
# iso to ...
unix_1 = timeutil.iso_to_unix(iso)
utc_date_1 = timeutil.iso_to_utc_date(iso)
local_date_1 = timeutil.iso_to_local_date(iso)
print('iso conversion:', unix_1)
print('iso conversion:', utc_date_1)
print('iso conversion:', local_date_1)
# utc_date to ...
unix_2 = timeutil.utc_date_to_unix(utc_date)
iso_2 = timeutil.utc_date_to_iso(utc_date)
local_date_2 = timeutil.utc_date_to_local_date(utc_date)
print('utc_date conversion:', unix_2)
print('utc_date conversion:', iso_2)
print('utc_date conversion:', local_date_2)
# local_date to ...
unix_3 = timeutil.local_date_to_unix(local_date)