def trades(self, symbol, start=None, end=None, retry=None, retry_wait=10):
symbol = pair_std_to_exchange(symbol, self.ID)
@request_retry(self.ID, retry, retry_wait)
def helper(s=None, e=None):
data = self._get("returnTradeHistory", {'currencyPair': symbol, 'start': s, 'end': e})
data.reverse()
return data
if not start:
yield map(lambda x: self._trade_normalize(x, symbol), helper())
else:
if not end:
end = pd.Timestamp.utcnow()
start = API._timestamp(start)
end = API._timestamp(end) - pd.Timedelta(nanoseconds=1)
start = int(start.timestamp())
end = int(end.timestamp())
s = start
e = start + 21600
while True:
if e > end:
e = end
yield map(lambda x: self._trade_normalize(x, symbol), helper(s=s, e=e))
s = e
e += 21600
if s >= end:
break
def _historical_trades(self, symbol, start_date, end_date, retry, retry_wait, freq='6H'):
symbol = self.std_symbol_to_exchange_symbol(symbol)
@request_retry(self.ID, retry, retry_wait)
def helper(start_date):
endpoint = f"{self.api}/public/Trades?symbol={symbol}&since={start_date}"
return requests.get(endpoint)
start_date = API._timestamp(start_date).timestamp() * 1000000000
end_date = API._timestamp(end_date).timestamp() * 1000000000
while start_date < end_date:
r = helper(start_date)
if r.status_code == 504 or r.status_code == 520:
# cloudflare gateway timeout or other error
time.sleep(60)
continue
elif r.status_code != 200:
self._handle_error(r, LOG)
else:
time.sleep(RATE_LIMIT_SLEEP)
data = r.json()
if 'error' in data and data['error']:
if data['error'] == ['EAPI:Rate limit exceeded']:
time.sleep(5)
continue
else:
raise Exception(f"Error processing URL {r.url}: {data['error']}")
yield data
start_date = int(data['result']['last'])
def trade_history(self, symbol: str = None, start=None, end=None):
params = {}
if start:
params['start'] = API._timestamp(start).timestamp()
if end:
params['end'] = API._timestamp(end).timestamp()
data = self._post_private('/private/TradesHistory', params)
if len(data['error']) != 0:
return data
ret = {}
for trade_id, trade in data['result']['trades'].items():
sym = self._convert_private_sym(trade['pair'])
std_sym = self.exchange_symbol_to_std_symbol(sym)
if symbol and self.exchange_symbol_to_std_symbol(sym) != symbol:
continue
# exception safety?
ret[trade_id] = {
'order_id': trade['ordertxid'],
'trade_id': trade_id,
'pair': std_sym,
'price': Decimal(trade['price']),
'amount': Decimal(trade['vol']),
'timestamp': trade['time'],
'side': SELL if trade['type'] == 'sell' else BUY,
'fee_currency': symbol.split('-')[1] if symbol else std_sym.split('-')[1],
'fee_amount': Decimal(trade['fee']),
'raw': trade
}
return ret
def trade_history(self, symbol: str, start=None, end=None):
payload = {
'currencyPair': self.info.std_symbol_to_exchange_symbol(symbol)
}
if start:
payload['start'] = API._timestamp(start).timestamp()
if end:
payload['end'] = API._timestamp(end).timestamp()
payload['limit'] = 10000
data = self._post("returnTradeHistory", payload)
ret = []
for trade in data:
ret.append({
'price': Decimal(trade['rate']),
'amount': Decimal(trade['amount']),
'timestamp': pd.Timestamp(trade['date']).timestamp(),
'side': BUY if trade['type'] == 'buy' else SELL,
'fee_currency': symbol.split('-')[1],
'fee_amount': Decimal(trade['fee']),
'trade_id': trade['tradeID'],
'order_id': trade['orderNumber']
})
return ret
def funding(self,
symbol: str,
start_date=None,
end_date=None,
retry=None,
retry_wait=10):
last = []
start = None
end = None
if end_date and not start_date:
start_date = '2019-01-01'
if start_date:
if not end_date:
end_date = pd.Timestamp.utcnow()
start = API._timestamp(start_date)
end = API._timestamp(end_date)
start = int(start.timestamp())
end = int(end.timestamp())
@request_retry(self.ID, retry, retry_wait)
def helper(start, end):
if start and end:
return requests.get(
f"{self.api}/funding_rates?future={symbol}&start_time={start}&end_time={end}"
)
else:
return requests.get(
f"{self.api}/funding_rates?symbol={symbol}")
while True:
r = helper(start, end)
if r.status_code == 429:
sleep(RATE_LIMIT_SLEEP)
continue
elif r.status_code == 500:
LOG.warning("%s: 500 for URL %s - %s", self.ID, r.url, r.text)
sleep(retry_wait)
continue
elif r.status_code != 200:
self._handle_error(r, LOG)
else:
sleep(RATE_LIMIT_SLEEP)
data = r.json()['result']
if data == []:
LOG.warning("%s: No data for range %d - %d", self.ID, start,
end)
else:
end = int(API._timestamp(data[-1]["time"]).timestamp()) + 1
orig_data = list(data)
# data = self._dedupe(data, last)
# last = list(orig_data)
data = [self._funding_normalization(x, symbol) for x in data]
return data
def _get_trades_hist(self, symbol, start_date, end_date, retry,
retry_wait):
last = []
start = None
end = None
if start_date:
if not end_date:
end_date = pd.Timestamp.utcnow()
start = API._timestamp(start_date)
end = API._timestamp(end_date) - pd.Timedelta(nanoseconds=1)
start = int(start.timestamp() * 1000)
end = int(end.timestamp() * 1000)
@request_retry(self.ID, retry, retry_wait)
def helper(start, end):
if start and end:
return requests.get(
f"{self.api}trades/{symbol}/hist?limit={REQUEST_LIMIT}&start={start}&end={end}&sort=1"
)
else:
return requests.get(f"{self.api}trades/{symbol}/hist")
while True:
r = helper(start, end)
if r.status_code == 429:
sleep(int(r.headers['Retry-After']))
continue
elif r.status_code == 500:
LOG.warning("%s: 500 for URL %s - %s", self.ID, r.url, r.text)
sleep(retry_wait)
continue
elif r.status_code != 200:
self._handle_error(r, LOG)
data = r.json()
if data == []:
LOG.warning("%s: No data for range %d - %d", self.ID, start,
end)
else:
if data[-1][1] == start:
LOG.warning(
"%s: number of trades exceeds exchange time window, some data will not be retrieved for time %d",
self.ID, start)
start += 1
else:
start = data[-1][1]
orig_data = list(data)
data = self._dedupe(data, last)
last = list(orig_data)
data = list(
map(lambda x: self._trade_normalization(symbol, x), data))
yield data
if len(orig_data) < REQUEST_LIMIT:
break
def _get_trades(self, instrument, start_date, end_date, retry, retry_wait):
start = None
end = None
if start_date:
if not end_date:
end_date = pd.Timestamp.utcnow()
start = API._timestamp(start_date)
end = API._timestamp(end_date) - pd.Timedelta(nanoseconds=1)
start = int(start.timestamp() * 1000)
end = int(end.timestamp() * 1000)
@request_retry(self.ID, retry, retry_wait)
def helper(start, end):
if start and end:
return requests.get(
f"{self.api}get_last_trades_by_instrument_and_time?&start_timestamp={start}&end_timestamp={end}&instrument_name={instrument}&include_old=true&count={REQUEST_LIMIT}"
)
else:
return requests.get(
f"{self.api}get_last_trades_by_instrument?instrument_name={instrument}&include_old=true&count={REQUEST_LIMIT}"
)
while True:
r = helper(start, end)
if r.status_code == 429:
sleep(int(r.headers['Retry-After']))
continue
elif r.status_code == 500:
LOG.warning("%s: 500 for URL %s - %s", self.ID, r.url, r.text)
sleep(retry_wait)
continue
elif r.status_code != 200:
self._handle_error(r, LOG)
else:
sleep(RATE_LIMIT_SLEEP)
data = r.json()["result"]["trades"]
if data == []:
LOG.warning("%s: No data for range %d - %d", self.ID, start,
end)
else:
if data[-1]["timestamp"] == start:
LOG.warning(
"%s: number of trades exceeds exchange time window, some data will not be retrieved for time %d",
self.ID, start)
start += 1
else:
start = data[-1]["timestamp"]
orig_data = data
data = [self._trade_normalization(x) for x in data]
yield data
if len(orig_data) < REQUEST_LIMIT or not start or not end:
break
def trade_history(self, symbol: str, start=None, end=None):
payload = {'currencyPair': pair_std_to_exchange(symbol, self.ID)}
if start:
payload['start'] = API._timestamp(start).timestamp()
if end:
payload['end'] = API._timestamp(end).timestamp()
payload['limit'] = 10000
return self._post("returnTradeHistory", payload)
def _funding_normalization(self, funding: dict, symbol: str) -> dict:
return {
'timestamp': API._timestamp(funding['time']).timestamp(),
'symbol': funding['future'],
'feed': self.ID,
'rate': funding['rate']
}
def _funding_normalization(self, funding: dict, symbol: str) -> dict:
ts = pd.to_datetime(funding['time'], format="%Y-%m-%dT%H:%M:%S%z")
return {
'timestamp': API._timestamp(funding['time']).timestamp(),
'pair': funding['future'],
'feed': self.ID,
'rate': funding['rate']
}
def _trade_normalization(self, trade: dict, symbol: str) -> dict:
return {
'timestamp': API._timestamp(trade['time']).timestamp(),
'symbol': symbol,
'id': trade['id'],
'feed': self.ID,
'side': SELL if trade['side'] == 'sell' else BUY,
'amount': trade['size'],
'price': trade['price']
}
def trades(self, symbol, start=None, end=None, retry=None, retry_wait=10):
"""
data format
{
'timestamp': '2018-01-01T23:59:59.907Z',
'symbol': 'XBTUSD',
'side': 'Buy',
'size': 1900,
'price': 13477,
'tickDirection': 'ZeroPlusTick',
'trdMatchID': '14fcc8d7-d056-768d-3c46-1fdf98728343',
'grossValue': 14098000,
'homeNotional': 0.14098,
'foreignNotional': 1900
}
"""
symbol = self.info.std_symbol_to_exchange_symbol(symbol)
d = dt.utcnow().date()
d -= timedelta(days=1)
rest_end_date = pd.Timestamp(dt(d.year, d.month, d.day))
start = API._timestamp(start) if start else start
end = API._timestamp(end) if end else end
rest_start = start
s3_scrape = False
if start:
if rest_end_date - pd.Timedelta(microseconds=1) > start:
rest_start = rest_end_date
s3_scrape = True
if s3_scrape:
rest_end_date -= pd.Timedelta(microseconds=1)
if API._timestamp(end) < rest_end_date:
rest_end_date = end
for data in self._scrape_s3(symbol, 'trade', start, rest_end_date):
yield list(map(self._s3_data_normalization, data))
if end is None or end > rest_end_date:
for data in self._get('trade', symbol, rest_start, end, retry,
retry_wait):
yield list(map(self._trade_normalization, data))
def ledger(self,
aclass=None,
asset=None,
ledger_type=None,
start=None,
end=None):
params = {}
if start:
params['start'] = API._timestamp(start).timestamp()
if end:
params['end'] = API._timestamp(end).timestamp()
if aclass:
params['aclass'] = aclass
if asset:
params['asset'] = asset
if ledger_type:
params['type'] = ledger_type
data = self._post_private('/private/Ledgers', params)
if len(data['error']) != 0:
return data
ret = {}
for ledger_id, ledger in data['result']['ledger'].items():
sym = self._convert_private_sym(ledger['asset'])
ret[ledger_id] = {
'ref_id': ledger['refid'],
'ledger_id': ledger_id,
'type': ledger['type'],
'sub_type': ledger['subtype'],
'asset': sym,
'asset_class': ledger['aclass'],
'amount': Decimal(ledger['amount']),
'balance': Decimal(ledger['balance']),
'timestamp': ledger['time'],
'fee_currency': sym,
'fee_amount': Decimal(ledger['fee']),
'raw': ledger
}
return ret
def _get(self, ep, symbol, start_date, end_date, retry, retry_wait, freq='6H'):
dates = [None]
if start_date:
if not end_date:
end_date = pd.Timestamp.utcnow()
dates = pd.interval_range(API._timestamp(start_date), API._timestamp(end_date), freq=freq).tolist()
if len(dates) == 0:
dates.append(pd.Interval(left=API._timestamp(start_date), right=API._timestamp(end_date)))
elif dates[-1].right < API._timestamp(end_date):
dates.append(pd.Interval(dates[-1].right, API._timestamp(end_date)))
@request_retry(self.ID, retry, retry_wait)
def helper(start, start_date, end_date):
if start_date and end_date:
endpoint = f'/api/v1/{ep}?symbol={symbol}&count={API_MAX}&reverse=false&start={start}&startTime={start_date}&endTime={end_date}'
else:
endpoint = f'/api/v1/{ep}?symbol={symbol}&reverse=true'
header = {}
if self.key_id and self.key_secret:
header = self._generate_signature("GET", endpoint)
header['Accept'] = 'application/json'
return requests.get('{}{}'.format(self.api, endpoint), headers=header)
for interval in dates:
start = 0
if interval is not None:
end = interval.right
end -= pd.Timedelta(nanoseconds=1)
start_date = str(interval.left).replace(" ", "T") + "Z"
end_date = str(end).replace(" ", "T") + "Z"
while True:
r = helper(start, start_date, end_date)
if r.status_code in {502, 504}:
LOG.warning("%s: %d for URL %s - %s", self.ID, r.status_code, r.url, r.text)
sleep(retry_wait)
continue
elif r.status_code == 429:
sleep(API_REFRESH)
continue
elif r.status_code != 200:
self._handle_error(r, LOG)
else:
sleep(RATE_LIMIT_SLEEP)
limit = int(r.headers['X-RateLimit-Remaining'])
data = r.json()
yield data
if len(data) != API_MAX:
break
if limit < 1:
sleep(API_REFRESH)
start += len(data)
def get_trades_history(self, symbol: str, start=None, end=None):
params = {}
if start:
params['start'] = API._timestamp(start).timestamp()
if end:
params['end'] = API._timestamp(end).timestamp()
data = self._post_private('/private/TradesHistory', params)
if len(data['error']) != 0:
return data
ret = []
for trade_id, trade in data['result']['trades'].items():
sym = trade['pair']
sym = sym.replace('XX', 'X')
sym = sym.replace('ZUSD', 'USD')
sym = sym.replace('ZCAD', 'CAD')
sym = sym.replace('ZEUR', 'EUR')
sym = sym.replace('ZGBP', 'GBP')
sym = sym.replace('ZJPY', 'JPY')
if symbol_exchange_to_std(sym) != symbol:
continue
ret.append({
'price': Decimal(trade['price']),
'amount': Decimal(trade['vol']),
'timestamp': trade['time'],
'side': SELL if trade['type'] == 'sell' else BUY,
'fee_currency': symbol.split('-')[1],
'fee_amount': Decimal(trade['fee']),
'trade_id': trade_id,
'order_id': trade['ordertxid']
})
return ret
def trade_history(self, symbol: str, start=None, end=None):
sym = symbol_std_to_exchange(symbol, self.ID)
params = {'symbol': sym, 'limit_trades': 500}
if start:
params['timestamp'] = API._timestamp(start).timestamp()
data = self._post("/v1/mytrades", params)
return [{
'price': Decimal(trade['price']),
'amount': Decimal(trade['amount']),
'timestamp': trade['timestampms'] / 1000,
'side': BUY if trade['type'].lower() == 'buy' else SELL,
'fee_currency': trade['fee_currency'],
'fee_amount': trade['fee_amount'],
'trade_id': trade['tid'],
'order_id': trade['order_id']
} for trade in data]
