def ticker(self, symbol: str, retry=None, retry_wait=0):
sym = symbol_std_to_exchange(symbol, self.ID)
data = self._get(f"/v1/pubticker/{sym}", retry, retry_wait)
return {'symbol': symbol,
'feed': self.ID,
'bid': Decimal(data['bid']),
'ask': Decimal(data['ask'])
}
def trades(self,
symbol: str,
start=None,
end=None,
retry=None,
retry_wait=10):
symbol = symbol_std_to_exchange(symbol, self.ID)
for data in self._get_trades(symbol, start, end, retry, retry_wait):
yield data
def ticker(self, symbol: str, retry=None, retry_wait=10):
sym = symbol_std_to_exchange(symbol, self.ID)
data = self._get("returnTicker", retry=retry, retry_wait=retry_wait)
return {
'symbol': symbol,
'feed': self.ID,
'bid': Decimal(data[sym]['lowestAsk']),
'ask': Decimal(data[sym]['highestBid'])
}
def l2_book(self, symbol: str, retry=None, retry_wait=10):
ret = {symbol: {BID: sd(), ASK: sd()}}
data = next(
self._get('orderBook/L2', symbol_std_to_exchange(symbol, self.ID),
None, None, retry, retry_wait))
for update in data:
side = ASK if update['side'] == 'Sell' else BID
ret[symbol][side][update['price']] = update['size']
return ret
def ticker(self, symbol: str, retry=None, retry_wait=0):
sym = symbol_std_to_exchange(symbol, self.ID)
data = self._get(f"/markets/{sym}", retry=retry, retry_wait=retry_wait)
return {
'symbol': symbol,
'feed': self.ID,
'bid': data['result']['bid'],
'ask': data['result']['ask']
}
def l2_book(self, symbol: str, retry=None, retry_wait=0):
sym = symbol_std_to_exchange(symbol, self.ID)
data = self._get("returnOrderBook", {'currencyPair': sym},
retry=retry,
retry_wait=retry_wait)
return {
BID: sd({Decimal(u[0]): Decimal(u[1])
for u in data['bids']}),
ASK: sd({Decimal(u[0]): Decimal(u[1])
for u in data['asks']})
}
def l2_book(self, symbol: str, retry=None, retry_wait=0):
sym = symbol_std_to_exchange(symbol, self.ID)
data = self._get(f"/markets/{sym}/orderbook", {'depth': 100},
retry=retry,
retry_wait=retry_wait)
return {
BID: sd({u[0]: u[1]
for u in data['result']['bids']}),
ASK: sd({u[0]: u[1]
for u in data['result']['asks']})
}
def l2_book(self, symbol: str, retry=None, retry_wait=0):
sym = symbol_std_to_exchange(symbol, self.ID)
data = self._get(f"/v1/book/{sym}", retry, retry_wait)
return {
BID: sd({
Decimal(u['price']): Decimal(u['amount'])
for u in data['bids']
}),
ASK: sd({
Decimal(u['price']): Decimal(u['amount'])
for u in data['asks']
})
}
async def _book(self, msg: dict, timestamp: float):
"""
{
'ch':'market.BTC_CW.depth.step0',
'ts':1565857755564,
'tick':{
'mrid':14848858327,
'id':1565857755,
'bids':[
[ Decimal('9829.99'), 1], ...
]
'asks':[
[ 9830, 625], ...
]
},
'ts':1565857755552,
'version':1565857755,
'ch':'market.BTC_CW.depth.step0'
}
"""
pair = symbol_std_to_exchange(msg['ch'].split('.')[1], self.id)
data = msg['tick']
forced = pair not in self.l2_book
# When Huobi Delists pairs, empty updates still sent:
# {'ch': 'market.AKRO-USD.depth.step0', 'ts': 1606951241196, 'tick': {'mrid': 50651100044, 'id': 1606951241, 'ts': 1606951241195, 'version': 1606951241, 'ch': 'market.AKRO-USD.depth.step0'}}
# {'ch': 'market.AKRO-USD.depth.step0', 'ts': 1606951242297, 'tick': {'mrid': 50651100044, 'id': 1606951242, 'ts': 1606951242295, 'version': 1606951242, 'ch': 'market.AKRO-USD.depth.step0'}}
if 'bids' in data and 'asks' in data:
update = {
BID:
sd({
Decimal(price): Decimal(amount)
for price, amount in data['bids']
}),
ASK:
sd({
Decimal(price): Decimal(amount)
for price, amount in data['asks']
})
}
if not forced:
self.previous_book[pair] = self.l2_book[pair]
self.l2_book[pair] = update
await self.book_callback(self.l2_book[pair], L2_BOOK, pair, forced,
False,
timestamp_normalize(self.id,
msg['ts']), timestamp)
def ticker(self, symbol: str, retry=None, retry_wait=0):
sym = symbol_std_to_exchange(symbol, self.ID + 'REST')
data = self._post_public("/public/Ticker",
payload={'pair': sym},
retry=retry,
retry_wait=retry_wait)
data = data['result']
for _, val in data.items():
return {
'symbol': symbol,
'feed': self.ID,
'bid': Decimal(val['b'][0]),
'ask': Decimal(val['a'][0])
}
def l2_book(self, symbol: str, retry=None, retry_wait=0):
sym = symbol_std_to_exchange(symbol, self.ID + 'REST')
data = self._post_public("/public/Depth", {
'pair': sym,
'count': 200
},
retry=retry,
retry_wait=retry_wait)
for _, val in data['result'].items():
return {
BID: sd({Decimal(u[0]): Decimal(u[1])
for u in val['bids']}),
ASK: sd({Decimal(u[0]): Decimal(u[1])
for u in val['asks']})
}
def trades(self,
symbol: str,
start=None,
end=None,
retry=None,
retry_wait=10):
sym = symbol_std_to_exchange(symbol, self.ID)
params = {'limit_trades': 500}
if start:
params['since'] = int(pd.Timestamp(start).timestamp() * 1000)
if end:
end_ts = int(pd.Timestamp(end).timestamp() * 1000)
def _trade_normalize(trade):
return {
'feed': self.ID,
'order_id': trade['tid'],
'symbol': sym,
'side': trade['type'],
'amount': Decimal(trade['amount']),
'price': Decimal(trade['price']),
'timestamp': trade['timestampms'] / 1000.0
}
while True:
data = reversed(
self._get(f"/v1/trades/{sym}?",
retry,
retry_wait,
params=params))
if end:
data = [
_trade_normalize(d) for d in data
if d['timestampms'] <= end_ts
]
else:
data = [_trade_normalize(d) for d in data]
yield data
if start:
params['since'] = int(data[-1]['timestamp'] * 1000) + 1
if len(data) < 500:
break
if not start and not end:
break
# GEMINI rate limits to 120 requests a minute
sleep(RATE_LIMIT_SLEEP)
def trade_history(self, symbol: str, start=None, end=None):
sym = symbol_std_to_exchange(symbol, self.ID)
params = {'symbol': sym, 'limit_trades': 500}
if start:
params['timestamp'] = API._timestamp(start).timestamp()
data = self._post("/v1/mytrades", params)
return [{
'price': Decimal(trade['price']),
'amount': Decimal(trade['amount']),
'timestamp': trade['timestampms'] / 1000,
'side': BUY if trade['type'].lower() == 'buy' else SELL,
'fee_currency': trade['fee_currency'],
'fee_amount': trade['fee_amount'],
'trade_id': trade['tid'],
'order_id': trade['order_id']
} for trade in data]
def trades(self, symbol, start=None, end=None, retry=None, retry_wait=10):
"""
data format
{
'timestamp': '2018-01-01T23:59:59.907Z',
'symbol': 'XBTUSD',
'side': 'Buy',
'size': 1900,
'price': 13477,
'tickDirection': 'ZeroPlusTick',
'trdMatchID': '14fcc8d7-d056-768d-3c46-1fdf98728343',
'grossValue': 14098000,
'homeNotional': 0.14098,
'foreignNotional': 1900
}
"""
symbol = symbol_std_to_exchange(symbol, self.ID)
d = dt.utcnow().date()
d -= timedelta(days=1)
rest_end_date = pd.Timestamp(dt(d.year, d.month, d.day))
start = API._timestamp(start) if start else start
end = API._timestamp(end) if end else end
rest_start = start
s3_scrape = False
if start:
if rest_end_date - pd.Timedelta(microseconds=1) > start:
rest_start = rest_end_date
s3_scrape = True
if s3_scrape:
rest_end_date -= pd.Timedelta(microseconds=1)
if API._timestamp(end) < rest_end_date:
rest_end_date = end
for data in self._scrape_s3(symbol, 'trade', start, rest_end_date):
yield list(map(self._s3_data_normalization, data))
if end is None or end > rest_end_date:
for data in self._get('trade', symbol, rest_start, end, retry,
retry_wait):
yield list(map(self._trade_normalization, data))
def place_order(self, symbol: str, side: str, order_type: str, amount: Decimal, price=None, client_order_id=None, options=None):
if not price:
raise ValueError('Gemini only supports limit orders, must specify price')
ot = normalize_trading_options(self.ID, order_type)
sym = symbol_std_to_exchange(symbol, self.ID)
parameters = {
'type': ot,
'symbol': sym,
'side': side,
'amount': str(amount),
'price': str(price),
'options': [normalize_trading_options(self.ID, o) for o in options] if options else []
}
if client_order_id:
parameters['client_order_id'] = client_order_id
data = self._post("/v1/order/new", parameters)
return Gemini._order_status(data)
def _historical_trades(self,
symbol,
start_date,
end_date,
retry,
retry_wait,
freq='6H'):
symbol = symbol_std_to_exchange(symbol, self.ID + 'REST')
@request_retry(self.ID, retry, retry_wait)
def helper(start_date):
endpoint = f"{self.api}/public/Trades?symbol={symbol}&since={start_date}"
return requests.get(endpoint)
start_date = API._timestamp(start_date).timestamp() * 1000000000
end_date = API._timestamp(end_date).timestamp() * 1000000000
while start_date < end_date:
r = helper(start_date)
if r.status_code == 504 or r.status_code == 520:
# cloudflare gateway timeout or other error
time.sleep(60)
continue
elif r.status_code != 200:
self._handle_error(r, LOG)
else:
time.sleep(RATE_LIMIT_SLEEP)
data = r.json()
if 'error' in data and data['error']:
if data['error'] == ['EAPI:Rate limit exceeded']:
time.sleep(5)
continue
else:
raise Exception(
f"Error processing URL {r.url}: {data['error']}")
yield data
start_date = int(data['result']['last'])
def trades(self,
symbol: str,
start=None,
end=None,
retry=None,
retry_wait=10):
if start:
if not end:
end = pd.Timestamp.utcnow()
for data in self._historical_trades(symbol, start, end, retry,
retry_wait):
yield list(
map(lambda x: self._trade_normalization(x, symbol),
data['result'][next(iter(data['result']))]))
else:
sym = symbol_std_to_exchange(symbol, self.ID + 'REST')
data = self._post_public("/public/Trades", {'pair': sym},
retry=retry,
retry_wait=retry_wait)
data = data['result']
data = data[list(data.keys())[0]]
yield [self._trade_normalization(d, symbol) for d in data]
def trades(self, symbol, start=None, end=None, retry=None, retry_wait=10):
symbol = symbol_std_to_exchange(symbol, self.ID)
@request_retry(self.ID, retry, retry_wait)
def helper(s=None, e=None):
data = self._get("returnTradeHistory", {
'currencyPair': symbol,
'start': s,
'end': e
})
data.reverse()
return data
if not start:
yield map(lambda x: self._trade_normalize(x, symbol), helper())
else:
if not end:
end = pd.Timestamp.utcnow()
start = API._timestamp(start)
end = API._timestamp(end) - pd.Timedelta(nanoseconds=1)
start = int(start.timestamp())
end = int(end.timestamp())
s = start
e = start + 21600
while True:
if e > end:
e = end
yield map(lambda x: self._trade_normalize(x, symbol),
helper(s=s, e=e))
s = e
e += 21600
if s >= end:
break
def place_order(self,
symbol: str,
side: str,
order_type: str,
amount: Decimal,
price=None,
options=None):
if not price:
raise ValueError(
'Poloniex only supports limit orders, must specify price')
# Poloniex only supports limit orders, so check the order type
_ = normalize_trading_options(self.ID, order_type)
parameters = {}
if options:
parameters = {
normalize_trading_options(self.ID, o): 1
for o in options
}
parameters['currencyPair'] = symbol_std_to_exchange(symbol, self.ID)
parameters['amount'] = str(amount)
parameters['rate'] = str(price)
endpoint = None
if side == BUY:
endpoint = 'buy'
elif side == SELL:
endpoint = 'sell'
data = self._post(endpoint, parameters)
order = self.order_status(data['orderNumber'])
if 'error' not in order:
if len(data['resultingTrades']) == 0:
return order
else:
return Poloniex._trade_status(data['resultingTrades'], symbol,
data['orderNumber'], amount)
return data
async def _trade(self, msg: dict, timestamp: float):
"""
{
'ch': 'market.btcusd.trade.detail',
'ts': 1549773923965,
'tick': {
'id': 100065340982,
'ts': 1549757127140,
'data': [{'id': '10006534098224147003732', 'amount': Decimal('0.0777'), 'price': Decimal('3669.69'), 'direction': 'buy', 'ts': 1549757127140}]}
}
"""
for trade in msg['tick']['data']:
await self.callback(
TRADES,
feed=self.id,
symbol=symbol_std_to_exchange(msg['ch'].split('.')[1],
self.id),
order_id=trade['id'],
side=BUY if trade['direction'] == 'buy' else SELL,
amount=Decimal(trade['amount']),
price=Decimal(trade['price']),
timestamp=timestamp_normalize(self.id, trade['ts']),
receipt_timestamp=timestamp)
def trade_history(self, symbol: str, start=None, end=None):
payload = {'currencyPair': symbol_std_to_exchange(symbol, self.ID)}
if start:
payload['start'] = API._timestamp(start).timestamp()
if end:
payload['end'] = API._timestamp(end).timestamp()
payload['limit'] = 10000
data = self._post("returnTradeHistory", payload)
ret = []
for trade in data:
ret.append({
'price': Decimal(trade['rate']),
'amount': Decimal(trade['amount']),
'timestamp': pd.Timestamp(trade['date']).timestamp(),
'side': BUY if trade['type'] == 'buy' else SELL,
'fee_currency': symbol.split('-')[1],
'fee_amount': Decimal(trade['fee']),
'trade_id': trade['tradeID'],
'order_id': trade['orderNumber']
})
return ret
def test_bitcoincom_symbol_conversions():
load_exchange_symbol_mapping(BITCOINCOM)
for _, symbol in bitcoincom_symbols().items():
std = symbol_exchange_to_std(symbol)
assert symbol == symbol_std_to_exchange(std, BITCOINCOM)
def __init__(self,
address: Union[dict, str],
sandbox=False,
symbols=None,
channels=None,
subscription=None,
config: Union[Config, dict, str] = None,
callbacks=None,
max_depth=None,
book_interval=1000,
snapshot_interval=False,
checksum_validation=False,
cross_check=False,
origin=None):
"""
address: str, or dict
address to be used to create the connection.
The address protocol (wss or https) will be used to determine the connection type.
Use a "str" to pass one single address, or a dict of option/address
sandbox: bool
For authenticated channels, run against the sandbox websocket (when True)
max_depth: int
Maximum number of levels per side to return in book updates
book_interval: int
Number of updates between snapshots. Only applicable when book deltas are enabled.
Book deltas are enabled by subscribing to the book delta callback.
snapshot_interval: bool/int
Number of updates between snapshots. Only applicable when book delta is not enabled.
Updates between snapshots are not delivered to the client
checksum_validation: bool
Toggle checksum validation, when supported by an exchange.
cross_check: bool
Toggle a check for a crossed book. Should not be needed on exchanges that support
checksums or provide message sequence numbers.
origin: str
Passed into websocket connect. Sets the origin header.
"""
if isinstance(config, Config):
LOG.info(
'%s: reuse object Config containing the following main keys: %s',
self.id, ", ".join(config.config.keys()))
self.config = config
else:
LOG.info('%s: create Config from type: %r', self.id, type(config))
self.config = Config(config)
self.subscription = defaultdict(set)
self.address = address
self.book_update_interval = book_interval
self.snapshot_interval = snapshot_interval
self.cross_check = cross_check
self.updates = defaultdict(int)
self.do_deltas = False
self.symbols = []
self.normalized_symbols = []
self.channels = []
self.max_depth = max_depth
self.previous_book = defaultdict(dict)
self.origin = origin
self.checksum_validation = checksum_validation
self.ws_defaults = {
'ping_interval': 10,
'ping_timeout': None,
'max_size': 2**23,
'max_queue': None,
'origin': self.origin
}
self.key_id = os.environ.get(f'CF_{self.id}_KEY_ID') or self.config[
self.id.lower()].key_id
self.key_secret = os.environ.get(
f'CF_{self.id}_KEY_SECRET') or self.config[
self.id.lower()].key_secret
self._feed_config = defaultdict(list)
load_exchange_symbol_mapping(self.id, key_id=self.key_id)
if subscription is not None and (symbols is not None
or channels is not None):
raise ValueError(
"Use subscription, or channels and symbols, not both")
if subscription is not None:
for channel in subscription:
chan = feed_to_exchange(self.id, channel)
if is_authenticated_channel(channel):
if not self.key_id or not self.key_secret:
raise ValueError(
"Authenticated channel subscribed to, but no auth keys provided"
)
self.normalized_symbols.extend(subscription[channel])
self.subscription[chan].update([
symbol_std_to_exchange(symbol, self.id)
for symbol in subscription[channel]
])
self._feed_config[channel].extend(self.normalized_symbols)
if symbols:
self.normalized_symbols = symbols
self.symbols = [
symbol_std_to_exchange(symbol, self.id) for symbol in symbols
]
if channels:
self.channels = list(
set([feed_to_exchange(self.id, chan) for chan in channels]))
[
self._feed_config[channel].extend(self.normalized_symbols)
for channel in channels
]
if any(is_authenticated_channel(chan) for chan in channels):
if not self.key_id or not self.key_secret:
raise ValueError(
"Authenticated channel subscribed to, but no auth keys provided"
)
self._feed_config = dict(self._feed_config)
self.l3_book = {}
self.l2_book = {}
self.callbacks = {
FUNDING: Callback(None),
FUTURES_INDEX: Callback(None),
L2_BOOK: Callback(None),
L3_BOOK: Callback(None),
LIQUIDATIONS: Callback(None),
OPEN_INTEREST: Callback(None),
MARKET_INFO: Callback(None),
TICKER: Callback(None),
TRADES: Callback(None),
TRANSACTIONS: Callback(None),
VOLUME: Callback(None),
CANDLES: Callback(None),
ORDER_INFO: Callback(None)
}
if callbacks:
for cb_type, cb_func in callbacks.items():
self.callbacks[cb_type] = cb_func
if cb_type == BOOK_DELTA:
self.do_deltas = True
for key, callback in self.callbacks.items():
if not isinstance(callback, list):
self.callbacks[key] = [callback]
def test_coinbase_symbol_conversions():
load_exchange_symbol_mapping(COINBASE)
for _, symbol in coinbase_symbols().items():
assert symbol_exchange_to_std(symbol) == symbol_std_to_exchange(
symbol, COINBASE)
def test_poloniex_symbol_conversions():
load_exchange_symbol_mapping(POLONIEX)
for _, symbol in poloniex_symbols().items():
std = symbol_exchange_to_std(symbol)
assert symbol == symbol_std_to_exchange(std, POLONIEX)
def test_bitfinex_symbol_conversions():
load_exchange_symbol_mapping(BITFINEX)
for _, symbol in bitfinex_symbols().items():
std = symbol_exchange_to_std(symbol)
assert symbol == symbol_std_to_exchange(std, BITFINEX)
def test_hitbtc_symbol_conversions():
load_exchange_symbol_mapping(HITBTC)
for _, symbol in hitbtc_symbols().items():
std = symbol_exchange_to_std(symbol)
assert symbol == symbol_std_to_exchange(std, HITBTC)
def test_blockchain_symbol_conversions():
load_exchange_symbol_mapping(BLOCKCHAIN)
for _, symbol in blockchain_symbols().items():
assert symbol_exchange_to_std(symbol) == symbol_std_to_exchange(
symbol, BLOCKCHAIN)
def test_gemini_symbol_conversions():
load_exchange_symbol_mapping(GEMINI)
for normalized, symbol in gemini_symbols().items():
std = symbol_exchange_to_std(symbol)
assert symbol == symbol_std_to_exchange(std, GEMINI)
assert normalized == std
def test_bitstamp_symbol_conversions():
load_exchange_symbol_mapping(BITSTAMP)
for _, symbol in bitstamp_symbols().items():
std = symbol_exchange_to_std(symbol)
assert symbol == symbol_std_to_exchange(std, BITSTAMP)
