def trades(self, symbol, start=None, end=None, retry=None, retry_wait=10): symbol = pair_std_to_exchange(symbol, self.ID) @request_retry(self.ID, retry, retry_wait) def helper(s=None, e=None): data = self._get("returnTradeHistory", {'currencyPair': symbol, 'start': s, 'end': e}) data.reverse() return data if not start: yield map(lambda x: self._trade_normalize(x, symbol), helper()) else: if not end: end = pd.Timestamp.utcnow() start = API._timestamp(start) end = API._timestamp(end) - pd.Timedelta(nanoseconds=1) start = int(start.timestamp()) end = int(end.timestamp()) s = start e = start + 21600 while True: if e > end: e = end yield map(lambda x: self._trade_normalize(x, symbol), helper(s=s, e=e)) s = e e += 21600 if s >= end: break
def _historical_trades(self, symbol, start_date, end_date, retry, retry_wait, freq='6H'): symbol = self.std_symbol_to_exchange_symbol(symbol) @request_retry(self.ID, retry, retry_wait) def helper(start_date): endpoint = f"{self.api}/public/Trades?symbol={symbol}&since={start_date}" return requests.get(endpoint) start_date = API._timestamp(start_date).timestamp() * 1000000000 end_date = API._timestamp(end_date).timestamp() * 1000000000 while start_date < end_date: r = helper(start_date) if r.status_code == 504 or r.status_code == 520: # cloudflare gateway timeout or other error time.sleep(60) continue elif r.status_code != 200: self._handle_error(r, LOG) else: time.sleep(RATE_LIMIT_SLEEP) data = r.json() if 'error' in data and data['error']: if data['error'] == ['EAPI:Rate limit exceeded']: time.sleep(5) continue else: raise Exception(f"Error processing URL {r.url}: {data['error']}") yield data start_date = int(data['result']['last'])
def trade_history(self, symbol: str = None, start=None, end=None): params = {} if start: params['start'] = API._timestamp(start).timestamp() if end: params['end'] = API._timestamp(end).timestamp() data = self._post_private('/private/TradesHistory', params) if len(data['error']) != 0: return data ret = {} for trade_id, trade in data['result']['trades'].items(): sym = self._convert_private_sym(trade['pair']) std_sym = self.exchange_symbol_to_std_symbol(sym) if symbol and self.exchange_symbol_to_std_symbol(sym) != symbol: continue # exception safety? ret[trade_id] = { 'order_id': trade['ordertxid'], 'trade_id': trade_id, 'pair': std_sym, 'price': Decimal(trade['price']), 'amount': Decimal(trade['vol']), 'timestamp': trade['time'], 'side': SELL if trade['type'] == 'sell' else BUY, 'fee_currency': symbol.split('-')[1] if symbol else std_sym.split('-')[1], 'fee_amount': Decimal(trade['fee']), 'raw': trade } return ret
def trade_history(self, symbol: str, start=None, end=None): payload = { 'currencyPair': self.info.std_symbol_to_exchange_symbol(symbol) } if start: payload['start'] = API._timestamp(start).timestamp() if end: payload['end'] = API._timestamp(end).timestamp() payload['limit'] = 10000 data = self._post("returnTradeHistory", payload) ret = [] for trade in data: ret.append({ 'price': Decimal(trade['rate']), 'amount': Decimal(trade['amount']), 'timestamp': pd.Timestamp(trade['date']).timestamp(), 'side': BUY if trade['type'] == 'buy' else SELL, 'fee_currency': symbol.split('-')[1], 'fee_amount': Decimal(trade['fee']), 'trade_id': trade['tradeID'], 'order_id': trade['orderNumber'] }) return ret
def funding(self, symbol: str, start_date=None, end_date=None, retry=None, retry_wait=10): last = [] start = None end = None if end_date and not start_date: start_date = '2019-01-01' if start_date: if not end_date: end_date = pd.Timestamp.utcnow() start = API._timestamp(start_date) end = API._timestamp(end_date) start = int(start.timestamp()) end = int(end.timestamp()) @request_retry(self.ID, retry, retry_wait) def helper(start, end): if start and end: return requests.get( f"{self.api}/funding_rates?future={symbol}&start_time={start}&end_time={end}" ) else: return requests.get( f"{self.api}/funding_rates?symbol={symbol}") while True: r = helper(start, end) if r.status_code == 429: sleep(RATE_LIMIT_SLEEP) continue elif r.status_code == 500: LOG.warning("%s: 500 for URL %s - %s", self.ID, r.url, r.text) sleep(retry_wait) continue elif r.status_code != 200: self._handle_error(r, LOG) else: sleep(RATE_LIMIT_SLEEP) data = r.json()['result'] if data == []: LOG.warning("%s: No data for range %d - %d", self.ID, start, end) else: end = int(API._timestamp(data[-1]["time"]).timestamp()) + 1 orig_data = list(data) # data = self._dedupe(data, last) # last = list(orig_data) data = [self._funding_normalization(x, symbol) for x in data] return data
def _get_trades_hist(self, symbol, start_date, end_date, retry, retry_wait): last = [] start = None end = None if start_date: if not end_date: end_date = pd.Timestamp.utcnow() start = API._timestamp(start_date) end = API._timestamp(end_date) - pd.Timedelta(nanoseconds=1) start = int(start.timestamp() * 1000) end = int(end.timestamp() * 1000) @request_retry(self.ID, retry, retry_wait) def helper(start, end): if start and end: return requests.get( f"{self.api}trades/{symbol}/hist?limit={REQUEST_LIMIT}&start={start}&end={end}&sort=1" ) else: return requests.get(f"{self.api}trades/{symbol}/hist") while True: r = helper(start, end) if r.status_code == 429: sleep(int(r.headers['Retry-After'])) continue elif r.status_code == 500: LOG.warning("%s: 500 for URL %s - %s", self.ID, r.url, r.text) sleep(retry_wait) continue elif r.status_code != 200: self._handle_error(r, LOG) data = r.json() if data == []: LOG.warning("%s: No data for range %d - %d", self.ID, start, end) else: if data[-1][1] == start: LOG.warning( "%s: number of trades exceeds exchange time window, some data will not be retrieved for time %d", self.ID, start) start += 1 else: start = data[-1][1] orig_data = list(data) data = self._dedupe(data, last) last = list(orig_data) data = list( map(lambda x: self._trade_normalization(symbol, x), data)) yield data if len(orig_data) < REQUEST_LIMIT: break
def _get_trades(self, instrument, start_date, end_date, retry, retry_wait): start = None end = None if start_date: if not end_date: end_date = pd.Timestamp.utcnow() start = API._timestamp(start_date) end = API._timestamp(end_date) - pd.Timedelta(nanoseconds=1) start = int(start.timestamp() * 1000) end = int(end.timestamp() * 1000) @request_retry(self.ID, retry, retry_wait) def helper(start, end): if start and end: return requests.get( f"{self.api}get_last_trades_by_instrument_and_time?&start_timestamp={start}&end_timestamp={end}&instrument_name={instrument}&include_old=true&count={REQUEST_LIMIT}" ) else: return requests.get( f"{self.api}get_last_trades_by_instrument?instrument_name={instrument}&include_old=true&count={REQUEST_LIMIT}" ) while True: r = helper(start, end) if r.status_code == 429: sleep(int(r.headers['Retry-After'])) continue elif r.status_code == 500: LOG.warning("%s: 500 for URL %s - %s", self.ID, r.url, r.text) sleep(retry_wait) continue elif r.status_code != 200: self._handle_error(r, LOG) else: sleep(RATE_LIMIT_SLEEP) data = r.json()["result"]["trades"] if data == []: LOG.warning("%s: No data for range %d - %d", self.ID, start, end) else: if data[-1]["timestamp"] == start: LOG.warning( "%s: number of trades exceeds exchange time window, some data will not be retrieved for time %d", self.ID, start) start += 1 else: start = data[-1]["timestamp"] orig_data = data data = [self._trade_normalization(x) for x in data] yield data if len(orig_data) < REQUEST_LIMIT or not start or not end: break
def trade_history(self, symbol: str, start=None, end=None): payload = {'currencyPair': pair_std_to_exchange(symbol, self.ID)} if start: payload['start'] = API._timestamp(start).timestamp() if end: payload['end'] = API._timestamp(end).timestamp() payload['limit'] = 10000 return self._post("returnTradeHistory", payload)
def _funding_normalization(self, funding: dict, symbol: str) -> dict: return { 'timestamp': API._timestamp(funding['time']).timestamp(), 'symbol': funding['future'], 'feed': self.ID, 'rate': funding['rate'] }
def _funding_normalization(self, funding: dict, symbol: str) -> dict: ts = pd.to_datetime(funding['time'], format="%Y-%m-%dT%H:%M:%S%z") return { 'timestamp': API._timestamp(funding['time']).timestamp(), 'pair': funding['future'], 'feed': self.ID, 'rate': funding['rate'] }
def _trade_normalization(self, trade: dict, symbol: str) -> dict: return { 'timestamp': API._timestamp(trade['time']).timestamp(), 'symbol': symbol, 'id': trade['id'], 'feed': self.ID, 'side': SELL if trade['side'] == 'sell' else BUY, 'amount': trade['size'], 'price': trade['price'] }
def trades(self, symbol, start=None, end=None, retry=None, retry_wait=10): """ data format { 'timestamp': '2018-01-01T23:59:59.907Z', 'symbol': 'XBTUSD', 'side': 'Buy', 'size': 1900, 'price': 13477, 'tickDirection': 'ZeroPlusTick', 'trdMatchID': '14fcc8d7-d056-768d-3c46-1fdf98728343', 'grossValue': 14098000, 'homeNotional': 0.14098, 'foreignNotional': 1900 } """ symbol = self.info.std_symbol_to_exchange_symbol(symbol) d = dt.utcnow().date() d -= timedelta(days=1) rest_end_date = pd.Timestamp(dt(d.year, d.month, d.day)) start = API._timestamp(start) if start else start end = API._timestamp(end) if end else end rest_start = start s3_scrape = False if start: if rest_end_date - pd.Timedelta(microseconds=1) > start: rest_start = rest_end_date s3_scrape = True if s3_scrape: rest_end_date -= pd.Timedelta(microseconds=1) if API._timestamp(end) < rest_end_date: rest_end_date = end for data in self._scrape_s3(symbol, 'trade', start, rest_end_date): yield list(map(self._s3_data_normalization, data)) if end is None or end > rest_end_date: for data in self._get('trade', symbol, rest_start, end, retry, retry_wait): yield list(map(self._trade_normalization, data))
def ledger(self, aclass=None, asset=None, ledger_type=None, start=None, end=None): params = {} if start: params['start'] = API._timestamp(start).timestamp() if end: params['end'] = API._timestamp(end).timestamp() if aclass: params['aclass'] = aclass if asset: params['asset'] = asset if ledger_type: params['type'] = ledger_type data = self._post_private('/private/Ledgers', params) if len(data['error']) != 0: return data ret = {} for ledger_id, ledger in data['result']['ledger'].items(): sym = self._convert_private_sym(ledger['asset']) ret[ledger_id] = { 'ref_id': ledger['refid'], 'ledger_id': ledger_id, 'type': ledger['type'], 'sub_type': ledger['subtype'], 'asset': sym, 'asset_class': ledger['aclass'], 'amount': Decimal(ledger['amount']), 'balance': Decimal(ledger['balance']), 'timestamp': ledger['time'], 'fee_currency': sym, 'fee_amount': Decimal(ledger['fee']), 'raw': ledger } return ret
def _get(self, ep, symbol, start_date, end_date, retry, retry_wait, freq='6H'): dates = [None] if start_date: if not end_date: end_date = pd.Timestamp.utcnow() dates = pd.interval_range(API._timestamp(start_date), API._timestamp(end_date), freq=freq).tolist() if len(dates) == 0: dates.append(pd.Interval(left=API._timestamp(start_date), right=API._timestamp(end_date))) elif dates[-1].right < API._timestamp(end_date): dates.append(pd.Interval(dates[-1].right, API._timestamp(end_date))) @request_retry(self.ID, retry, retry_wait) def helper(start, start_date, end_date): if start_date and end_date: endpoint = f'/api/v1/{ep}?symbol={symbol}&count={API_MAX}&reverse=false&start={start}&startTime={start_date}&endTime={end_date}' else: endpoint = f'/api/v1/{ep}?symbol={symbol}&reverse=true' header = {} if self.key_id and self.key_secret: header = self._generate_signature("GET", endpoint) header['Accept'] = 'application/json' return requests.get('{}{}'.format(self.api, endpoint), headers=header) for interval in dates: start = 0 if interval is not None: end = interval.right end -= pd.Timedelta(nanoseconds=1) start_date = str(interval.left).replace(" ", "T") + "Z" end_date = str(end).replace(" ", "T") + "Z" while True: r = helper(start, start_date, end_date) if r.status_code in {502, 504}: LOG.warning("%s: %d for URL %s - %s", self.ID, r.status_code, r.url, r.text) sleep(retry_wait) continue elif r.status_code == 429: sleep(API_REFRESH) continue elif r.status_code != 200: self._handle_error(r, LOG) else: sleep(RATE_LIMIT_SLEEP) limit = int(r.headers['X-RateLimit-Remaining']) data = r.json() yield data if len(data) != API_MAX: break if limit < 1: sleep(API_REFRESH) start += len(data)
def get_trades_history(self, symbol: str, start=None, end=None): params = {} if start: params['start'] = API._timestamp(start).timestamp() if end: params['end'] = API._timestamp(end).timestamp() data = self._post_private('/private/TradesHistory', params) if len(data['error']) != 0: return data ret = [] for trade_id, trade in data['result']['trades'].items(): sym = trade['pair'] sym = sym.replace('XX', 'X') sym = sym.replace('ZUSD', 'USD') sym = sym.replace('ZCAD', 'CAD') sym = sym.replace('ZEUR', 'EUR') sym = sym.replace('ZGBP', 'GBP') sym = sym.replace('ZJPY', 'JPY') if symbol_exchange_to_std(sym) != symbol: continue ret.append({ 'price': Decimal(trade['price']), 'amount': Decimal(trade['vol']), 'timestamp': trade['time'], 'side': SELL if trade['type'] == 'sell' else BUY, 'fee_currency': symbol.split('-')[1], 'fee_amount': Decimal(trade['fee']), 'trade_id': trade_id, 'order_id': trade['ordertxid'] }) return ret
def trade_history(self, symbol: str, start=None, end=None): sym = symbol_std_to_exchange(symbol, self.ID) params = {'symbol': sym, 'limit_trades': 500} if start: params['timestamp'] = API._timestamp(start).timestamp() data = self._post("/v1/mytrades", params) return [{ 'price': Decimal(trade['price']), 'amount': Decimal(trade['amount']), 'timestamp': trade['timestampms'] / 1000, 'side': BUY if trade['type'].lower() == 'buy' else SELL, 'fee_currency': trade['fee_currency'], 'fee_amount': trade['fee_amount'], 'trade_id': trade['tid'], 'order_id': trade['order_id'] } for trade in data]