async def _funding(self, pairs: Iterable):
"""
{
"success": true,
"result": [
{
"future": "BTC-PERP",
"rate": 0.0025,
"time": "2019-06-02T08:00:00+00:00"
}
]
}
"""
# do not send more than 30 requests per second: doing so will result in HTTP 429 errors
rate_limiter = 0.1
# funding rates do not change frequently
wait_time = 60
async with aiohttp.ClientSession() as session:
while True:
for pair in pairs:
if '-PERP' not in pair:
continue
async with session.get(f"https://ftx.com/api/funding_rates?future={pair}") as response:
data = await response.text()
data = json.loads(data, parse_float=Decimal)
last_update = self.funding.get(pair, None)
update = str(data['result'][0]['rate']) + str(data['result'][0]['time'])
if last_update and last_update == update:
continue
else:
self.funding[pair] = update
await self.callback(FUNDING, feed=self.id,
symbol=symbol_exchange_to_std(data['result'][0]['future']),
rate=data['result'][0]['rate'],
timestamp=timestamp_normalize(self.id, data['result'][0]['time']))
await asyncio.sleep(rate_limiter)
await asyncio.sleep(wait_time)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'error' in msg:
LOG.error("%s: Error from exchange: %s", self.id, msg)
return
chan_id = msg[0]
if chan_id == 1002:
# the ticker channel doesn't have sequence ids
# so it should be None, except for the subscription
# ack, in which case its 1
seq_id = msg[1]
if seq_id is None and self._channel_map[
msg[2][0]] in self.subscription[1002]:
await self._ticker(msg[2], timestamp)
elif chan_id < 1000:
# order book updates - the channel id refers to
# the trading pair being updated
seq_no = msg[1]
if chan_id not in self.seq_no:
self.seq_no[chan_id] = seq_no
elif self.seq_no[chan_id] + 1 != seq_no and msg[2][0][0] != 'i':
LOG.warning(
"%s: missing sequence number. Received %d, expected %d",
self.id, seq_no, self.seq_no[chan_id] + 1)
raise MissingSequenceNumber
self.seq_no[chan_id] = seq_no
if msg[2][0][0] == 'i':
del self.seq_no[chan_id]
symbol = self._channel_map[chan_id]
if symbol in self._trade_book_symbols:
await self._book(msg[2], chan_id, timestamp)
elif chan_id == 1010:
# heartbeat - ignore
pass
else:
LOG.warning('%s: Invalid message type %s', self.id, msg)
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
# Handle REST endpoint messages first
if 'openInterest' in msg:
return await self._open_interest(msg, timestamp)
# Handle account updates from User Data Stream
if self.requires_authentication:
msg_type = msg.get('e')
if msg_type == 'ACCOUNT_UPDATE':
await self._account_update(msg, timestamp)
elif msg_type == 'ORDER_TRADE_UPDATE':
await self._order_update(msg, timestamp)
return
# Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>}
# streamName is of format <symbol>@<channel>
pair, _ = msg['stream'].split('@', 1)
msg = msg['data']
pair = pair.upper()
msg_type = msg.get('e')
if msg_type == 'bookTicker':
await self._ticker(msg, timestamp)
elif msg_type == 'depthUpdate':
await self._book(msg, pair, timestamp)
elif msg_type == 'aggTrade':
await self._trade(msg, timestamp)
elif msg_type == 'forceOrder':
await self._liquidations(msg, timestamp)
elif msg_type == 'markPriceUpdate':
await self._funding(msg, timestamp)
elif msg['e'] == 'kline':
await self._candle(msg, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
async def message_handler(self, msg: str, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
# Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>}
# streamName is of format <symbol>@<channel>
pair, _ = msg['stream'].split('@', 1)
msg = msg['data']
pair = pair.upper()
if msg['e'] == 'depthUpdate':
await self._book(msg, pair, timestamp)
elif msg['e'] == 'aggTrade':
await self._trade(msg, timestamp)
elif msg['e'] == '24hrTicker':
await self._ticker(msg, timestamp)
elif msg['e'] == 'forceOrder':
await self._liquidations(msg, timestamp)
elif msg['e'] == 'markPriceUpdate':
await self._funding(msg, timestamp)
else:
LOG.warning("%s: Unexpected message received: %s", self.id, msg)
async def message_handler(self, msg: str, conn, timestamp: float):
msg_dict = json.loads(msg, parse_float=Decimal)
# As a first update after subscription, Deribit sends a notification with no data
if "testnet" in msg_dict.keys():
LOG.debug("%s: Test response from derbit accepted %s", self.id,
msg)
elif "ticker" == msg_dict["params"]["channel"].split(".")[0]:
await self._ticker(msg_dict, timestamp)
elif "trades" == msg_dict["params"]["channel"].split(".")[0]:
await self._trade(msg_dict, timestamp)
elif "book" == msg_dict["params"]["channel"].split(".")[0]:
# checking if we got full book or its update
# if it's update there is 'prev_change_id' field
if "prev_change_id" not in msg_dict["params"]["data"].keys():
await self._book_snapshot(msg_dict, timestamp)
elif "prev_change_id" in msg_dict["params"]["data"].keys():
await self._book_update(msg_dict, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def _funding(self, pairs):
"""
{
"status": "ok",
"data": {
"estimated_rate": "0.000100000000000000",
"funding_rate": "-0.000362360011416593",
"contract_code": "BTC-USD",
"symbol": "BTC",
"fee_asset": "BTC",
"funding_time": "1603872000000",
"next_funding_time": "1603900800000"
},
"ts": 1603866304635
}
"""
while True:
for pair in pairs:
data = await self.http_conn.read(f'https://api.hbdm.com/swap-api/v1/swap_funding_rate?contract_code={pair}')
data = json.loads(data, parse_float=Decimal)
received = time.time()
update = (data['data']['funding_rate'], self.timestamp_normalize(int(data['data']['next_funding_time'])))
if pair in self.funding_updates and self.funding_updates[pair] == update:
await asyncio.sleep(1)
continue
self.funding_updates[pair] = update
f = Funding(
self.id,
self.exchange_symbol_to_std_symbol(pair),
None,
Decimal(data['data']['funding_rate']),
self.timestamp_normalize(int(data['data']['next_funding_time'])),
self.timestamp_normalize(int(data['data']['funding_time'])),
predicted_rate=Decimal(data['data']['estimated_rate']),
raw=data
)
await self.callback(FUNDING, f, received)
await asyncio.sleep(0.1)
async def _snapshot(self, symbol: str):
url = f"https://api.kucoin.com/api/v2/market/orderbook/level2?symbol={symbol}"
data = await self.http_conn.read(url)
data = json.loads(data, parse_float=Decimal)
data = data['data']
self.seq_no[symbol] = int(data['sequence'])
self.l2_book[symbol] = {
BID:
sd({
Decimal(price): Decimal(amount)
for price, amount in data['bids']
}),
ASK:
sd({
Decimal(price): Decimal(amount)
for price, amount in data['asks']
})
}
timestamp = time.time()
await self.book_callback(self.l2_book[symbol], L2_BOOK, symbol, True,
None, timestamp, timestamp)
def trade_history(self, symbol: str, start=None, end=None):
endpoint = f"/orders?product_id={symbol}&status=done"
data = self._request("GET", endpoint, auth=True)
data = json.loads(data.text, parse_float=Decimal)
return [{
'order_id':
order['id'],
'trade_id':
order['id'],
'side':
BUY if order['side'] == 'buy' else SELL,
'price':
Decimal(order['executed_value']) / Decimal(order['filled_size']),
'amount':
Decimal(order['filled_size']),
'timestamp':
order['done_at'].timestamp(),
'fee_amount':
Decimal(order['fill_fees']),
'fee_currency':
symbol.split('-')[1]
} for order in data]
async def message_handler(self, msg: str, conn, timestamp: float):
# unzip message
msg = zlib.decompress(msg, 16 + zlib.MAX_WBITS)
msg = json.loads(msg, parse_float=Decimal)
# Huobi sends a ping evert 5 seconds and will disconnect us if we do not respond to it
if 'ping' in msg:
await conn.write(json.dumps({'pong': msg['ping']}))
elif 'status' in msg and msg['status'] == 'ok':
return
elif 'ch' in msg:
if 'trade' in msg['ch']:
await self._trade(msg, timestamp)
elif 'depth' in msg['ch']:
await self._book(msg, timestamp)
elif 'kline' in msg['ch']:
_, symbol, _, interval = msg['ch'].split(".")
await self._candles(msg, symbol, interval, timestamp)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def _liquidations(self, pairs: list):
last_update = {}
async with aiohttp.ClientSession() as session:
while True:
for pair in pairs:
if 'SWAP' in pair:
instrument_type = 'swap'
else:
instrument_type = 'futures'
for status in (0, 1):
end_point = f"{self.api}{instrument_type}/v3/instruments/{pair}/liquidation?status={status}&limit=100"
async with session.get(end_point) as response:
data = await response.text()
data = json.loads(data, parse_float=Decimal)
timestamp = time.time()
if len(data) == 0 or (len(data) > 0 and last_update.get(pair) == data[0]):
continue
for entry in data:
if entry == last_update.get(pair):
break
await self.callback(LIQUIDATIONS,
feed=self.id,
symbol=entry['instrument_id'],
side=BUY if entry['type'] == '3' else SELL,
leaves_qty=Decimal(entry['size']),
price=Decimal(entry['price']),
order_id=None,
status='filled' if status == 1 else 'unfilled',
timestamp=timestamp,
receipt_timestamp=timestamp
)
last_update[pair] = data[0]
await asyncio.sleep(0.1)
await asyncio.sleep(60)
async def trades(self,
symbol: str,
start=None,
end=None,
retry_count=1,
retry_delay=10):
symbol = self.std_symbol_to_exchange_symbol(symbol)
start, end = self._interval_normalize(start, end)
if start:
start = int(start * 1000)
end = int(end * 1000)
while True:
endpoint = f"{self.api}get_last_trades_by_instrument?instrument_name={symbol}&include_old=true&count=1000"
if start and end:
endpoint = f"{self.api}get_last_trades_by_instrument_and_time?&start_timestamp={start}&end_timestamp={end}&instrument_name={symbol}&include_old=true&count=1000"
data = await self.http_conn.read(endpoint,
retry_count=retry_count,
retry_delay=retry_delay)
data = json.loads(data, parse_float=Decimal)["result"]["trades"]
if data:
if data[-1]["timestamp"] == start:
LOG.warning(
"%s: number of trades exceeds exchange time window, some data will not be retrieved for time %d",
self.id, start)
start += 1
else:
start = data[-1]["timestamp"]
orig_data = data
data = [self._trade_normalization(x) for x in data]
yield data
if len(orig_data) < 1000 or not start or not end:
break
await asyncio.sleep(1 / self.request_limit)
async def message_handler(self, msg: str, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'product_id' in msg and 'sequence' in msg and ('full' in self.channels or ('full' in self.config and msg['product_id'] in self.config['full'])):
pair = pair_exchange_to_std(msg['product_id'])
if msg['sequence'] <= self.seq_no[pair]:
return
elif ('full' in self.channels or 'full' in self.config) and msg['sequence'] != self.seq_no[pair] + 1:
LOG.warning("%s: Missing sequence number detected for %s", self.id, pair)
LOG.warning("%s: Requesting book snapshot", self.id)
await self._book_snapshot(self.pairs or self.book_pairs)
return
self.seq_no[pair] = msg['sequence']
if 'type' in msg:
if msg['type'] == 'ticker':
await self._ticker(msg, timestamp)
elif msg['type'] == 'match' or msg['type'] == 'last_match':
await self._book_update(msg, timestamp)
elif msg['type'] == 'snapshot':
await self._pair_level2_snapshot(msg, timestamp)
elif msg['type'] == 'l2update':
await self._pair_level2_update(msg, timestamp)
elif msg['type'] == 'open':
await self._open(msg, timestamp)
elif msg['type'] == 'done':
await self._done(msg, timestamp)
elif msg['type'] == 'change':
await self._change(msg, timestamp)
elif msg['type'] == 'received':
pass
elif msg['type'] == 'activate':
pass
elif msg['type'] == 'subscriptions':
pass
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def _request(self,
method: str,
endpoint: str,
auth: bool = False,
body=None,
retry_count=1,
retry_delay=60):
api = self.sandbox_api if self.sandbox else self.api
header = None
if auth:
header = self._generate_signature(
endpoint, method, body=json.dumps(body) if body else '')
if method == "GET":
data = await self.http_conn.read(f'{api}{endpoint}', header=header)
elif method == 'POST':
data = await self.http_conn.write(f'{api}{endpoint}',
msg=body,
header=header)
elif method == 'DELETE':
data = await self.http_conn.delete(f'{api}{endpoint}',
header=header)
return json.loads(data, parse_float=Decimal)
async def _snapshot(self, symbol: str):
"""
{
"id": 2679059670,
"asks": [[price, amount], [...], ...],
"bids": [[price, amount], [...], ...]
}
"""
url = f'https://api.gateio.ws/api/v4/spot/order_book?currency_pair={symbol}&limit=100&with_id=true'
ret = await self.http_conn.read(url)
data = json.loads(ret, parse_float=Decimal)
symbol = self.exchange_symbol_to_std_symbol(symbol)
self.l2_book[symbol] = {}
self.last_update_id[symbol] = data['id']
self.l2_book[symbol][BID] = sd({
Decimal(price): Decimal(amount)
for price, amount in data['bids']
})
self.l2_book[symbol][ASK] = sd({
Decimal(price): Decimal(amount)
for price, amount in data['asks']
})
async def message_handler(self, msg: str, conn: AsyncConnection,
timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if isinstance(msg, list):
hb_skip = False
chan_handler = self.handlers.get(msg[0])
if chan_handler is None:
if msg[1] == 'hb':
hb_skip = True
else:
LOG.warning('%s: Unregistered channel ID in message %s',
conn.uuid, msg)
return
seq_no = msg[-1]
expected = self.seq_no[conn.uuid] + 1
if seq_no != expected:
LOG.warning(
'%s: missed message (sequence number) received %d, expected %d',
conn.uuid, seq_no, expected)
raise MissingSequenceNumber
self.seq_no[conn.uuid] = seq_no
if hb_skip:
return
await chan_handler(msg, timestamp)
elif 'event' not in msg:
LOG.warning('%s: Unexpected msg (missing event) from exchange: %s',
conn.uuid, msg)
elif msg['event'] == 'error':
LOG.error('%s: Error from exchange: %s', conn.uuid, msg)
elif msg['event'] in ('info', 'conf'):
LOG.info('%s: %s from exchange: %s', conn.uuid, msg['event'], msg)
elif 'chanId' in msg and 'symbol' in msg:
self.register_channel_handler(msg, conn)
else:
LOG.warning('%s: Unexpected msg from exchange: %s', conn.uuid, msg)
async def _open_interest(self, pairs: Iterable):
"""
{
"success": true,
"result": {
"volume": 1000.23,
"nextFundingRate": 0.00025,
"nextFundingTime": "2019-03-29T03:00:00+00:00",
"expirationPrice": 3992.1,
"predictedExpirationPrice": 3993.6,
"strikePrice": 8182.35,
"openInterest": 21124.583
}
}
"""
while True:
for pair in pairs:
# OI only for perp and futures, so check for / in pair name indicating spot
if '/' in pair:
continue
end_point = f"https://ftx.com/api/futures/{pair}/stats"
data = await self.http_conn.read(end_point)
received = time()
data = json.loads(data, parse_float=Decimal)
if 'result' in data:
oi = data['result']['openInterest']
if oi != self._open_interest_cache.get(pair, None):
o = OpenInterest(
self.id,
self.exchange_symbol_to_std_symbol(pair),
oi,
None,
raw=data)
await self.callback(OPEN_INTEREST, o, received)
self._open_interest_cache[pair] = oi
await asyncio.sleep(1)
await asyncio.sleep(60)
async def trades(self,
symbol: str,
start=None,
end=None,
retry_count=1,
retry_delay=60):
symbol = self.std_symbol_to_exchange_symbol(symbol)
start, end = self._interval_normalize(start, end)
if start and end:
start = int(start * 1000)
end = int(end * 1000)
while True:
if start and end:
endpoint = f"{self.api}aggTrades?symbol={symbol}&limit=1000&startTime={start}&endTime={end}"
else:
endpoint = f"{self.api}aggTrades?symbol={symbol}&limit=1000"
r = await self.http_conn.read(endpoint,
retry_count=retry_count,
retry_delay=retry_delay)
data = json.loads(r, parse_float=Decimal)
if data:
if data[-1]['T'] == start:
LOG.warning(
"%s: number of trades exceeds exchange time window, some data will not be retrieved for time %d",
self.id, start)
start += 1
else:
start = data[-1]['T']
yield [self._trade_normalization(symbol, d) for d in data]
if len(data) < 1000 or end is None:
break
await asyncio.sleep(1 / self.request_limit)
async def _funding(self, pairs):
async with aiohttp.ClientSession() as session:
while True:
for pair in pairs:
async with session.get(f'https://api.hbdm.com/swap-api/v1/swap_funding_rate?contract_code={pair}') as response:
data = await response.text()
data = json.loads(data, parse_float=Decimal)
received = time.time()
update = (data['data']['funding_rate'], timestamp_normalize(self.id, int(data['data']['next_funding_time'])))
if pair in self.funding_updates and self.funding_updates[pair] == update:
await asyncio.sleep(1)
continue
self.funding_updates[pair] = update
await self.callback(FUNDING,
feed=self.id,
symbol=pair,
timestamp=timestamp_normalize(self.id, data['ts']),
receipt_timestamp=received,
rate=Decimal(update[0]),
next_funding_time=update[1]
)
await asyncio.sleep(0.1)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'table' in msg:
if msg['table'] == 'trade':
await self._trade(msg, timestamp)
elif msg['table'] == 'order':
await self._order(msg, timestamp)
elif msg['table'] == 'orderBookL2':
await self._book(msg, timestamp)
elif msg['table'] == 'funding':
await self._funding(msg, timestamp)
elif msg['table'] == 'instrument':
await self._instrument(msg, timestamp)
elif msg['table'] == 'quote':
await self._ticker(msg, timestamp)
elif msg['table'] == 'liquidation':
await self._liquidation(msg, timestamp)
else:
LOG.warning("%s: Unhandled table=%r in %r", conn.uuid,
msg['table'], msg)
elif 'info' in msg:
LOG.debug("%s: Info message from exchange: %s", conn.uuid, msg)
elif 'subscribe' in msg:
if not msg['success']:
LOG.error("%s: Subscribe failure: %s", conn.uuid, msg)
elif 'error' in msg:
LOG.error("%s: Error message from exchange: %s", conn.uuid, msg)
elif 'request' in msg:
if msg['success']:
LOG.debug("%s: Success %s", conn.uuid,
msg['request'].get('op'))
else:
LOG.warning("%s: Failure %s", conn.uuid, msg['request'])
else:
LOG.warning("%s: Unexpected message from exchange: %s", conn.uuid,
msg)
async def trades(self, symbol: str, start=None, end=None, retry_count=1, retry_delay=60):
symbol = self.std_symbol_to_exchange_symbol(symbol)
last = []
start, end = self._interval_normalize(start, end)
while True:
endpoint = f"{self.api}/markets/{symbol}/trades"
if start and end:
endpoint = f"{self.api}/markets/{symbol}/trades?start_time={start}&end_time={end}"
r = await self.http_conn.read(endpoint, retry_count=retry_count, retry_delay=retry_delay)
data = json.loads(r, parse_float=Decimal)['result']
orig_data = list(data)
data = self._dedupe(data, last)
last = list(orig_data)
data = [self._trade_normalization(x, symbol) for x in data]
yield data
if len(orig_data) < 5000:
break
end = int(data[-1]['timestamp'])
await asyncio.sleep(1 / self.request_limit)
async def message_handler(self, msg: str, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if isinstance(msg, list):
if self.channel_map[msg[0]][0] == 'trade':
await self._trade(msg, self.channel_map[msg[0]][1], timestamp)
elif self.channel_map[msg[0]][0] == 'ticker':
await self._ticker(msg, self.channel_map[msg[0]][1], timestamp)
elif self.channel_map[msg[0]][0] == 'book':
await self._book(msg, self.channel_map[msg[0]][1], timestamp)
else:
LOG.warning("%s: No mapping for message %s", self.id, msg)
else:
if msg['event'] == 'heartbeat':
return
elif msg['event'] == 'systemStatus':
return
elif msg['event'] == 'subscriptionStatus' and msg[
'status'] == 'subscribed':
self.channel_map[msg['channelID']] = (
msg['subscription']['name'],
pair_exchange_to_std(msg['pair']))
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
def main(filename):
with open(filename, 'r') as fp:
counter = 0
for line in fp.readlines():
counter += 1
if line == "\n":
continue
if line.startswith("configuration"):
continue
start = line[:3]
if start == 'wss':
continue
if start == 'htt':
_, line = line.split(" -> ")
_, line = line.split(": ", 1)
if "header: " in line:
line = line.split("header:")[0]
try:
if 'OKCOIN' in filename or 'OKEX' in filename:
if line.startswith('b\'') or line.startswith('b"'):
line = bytes_string_to_bytes(line)
line = zlib.decompress(line, -15).decode()
elif 'HUOBI' in filename and 'ws' in filename:
line = bytes_string_to_bytes(line)
line = zlib.decompress(line, 16 + zlib.MAX_WBITS)
elif 'UPBIT' in filename:
if line.startswith('b\'') or line.startswith('b"'):
line = line.strip()[2:-1]
_ = json.loads(line)
except Exception:
print(f"Failed on line {counter}: ")
print(line)
raise
print(f"Successfully verified {counter} updates")
async def message_handler(self, msg: str, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if 'info' in msg:
LOG.info("%s - info message: %s", self.id, msg)
elif 'subscribe' in msg:
if not msg['success']:
LOG.error("%s: subscribe failed: %s", self.id, msg)
elif 'error' in msg:
LOG.error("%s: Error message from exchange: %s", self.id, msg)
else:
if msg['table'] == 'trade':
await self._trade(msg, timestamp)
elif msg['table'] == 'orderBookL2':
await self._book(msg, timestamp)
elif msg['table'] == 'funding':
await self._funding(msg, timestamp)
elif msg['table'] == 'instrument':
await self._instrument(msg, timestamp)
elif msg['table'] == 'quote':
await self._ticker(msg, timestamp)
elif msg['table'] == 'liquidation':
await self._liquidation(msg, timestamp)
else:
LOG.warning("%s: Unhandled message %s", self.id, msg)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg, parse_float=Decimal)
if isinstance(msg, list):
channel_id = msg[0]
if channel_id not in self.channel_map:
LOG.warning("%s: Invalid channel id received %d", self.id,
channel_id)
LOG.warning("%s: message with invalid channel id: %s", self.id,
msg)
LOG.warning("%s: channel map: %s", self.id, self.channel_map)
else:
channel, pair = self.channel_map[channel_id]
if channel == 'trade':
await self._trade(msg, pair, timestamp)
elif channel == 'ticker':
await self._ticker(msg, pair, timestamp)
elif channel == 'book':
await self._book(msg, pair, timestamp)
else:
LOG.warning("%s: No mapping for message %s", self.id, msg)
LOG.warning("%s: channel map: %s", self.id,
self.channel_map)
else:
if msg['event'] == 'heartbeat':
return
elif msg['event'] == 'systemStatus':
return
elif msg['event'] == 'subscriptionStatus' and msg[
'status'] == 'subscribed':
self.channel_map[msg['channelID']] = (
msg['subscription']['name'],
symbol_exchange_to_std(msg['pair']))
else:
LOG.warning("%s: Invalid message type %s", self.id, msg)
async def message_handler(self, msg: str, conn, timestamp: float):
msg = json.loads(msg)
await self._market_info(msg, timestamp)
def test_datetime_decode(value, expected):
assert yapic_json.loads(value, parse_date=True) == expected
bytes_value = value.encode("utf-8")
assert yapic_json.loads(bytes_value, parse_date=True) == expected
async def _get(self, command: str, retry_count, retry_delay, params=''):
api = self.api if not self.sandbox else self.sandbox_api
resp = await self.http_conn.read(f"{api}{command}{params}",
retry_count=retry_count,
retry_delay=retry_delay)
return json.loads(resp, parse_float=Decimal)
def _get(self,
ep,
symbol,
start_date,
end_date,
retry,
retry_wait,
freq='6H'):
dates = [None]
if start_date:
if not end_date:
end_date = pd.Timestamp.utcnow()
dates = pd.interval_range(API._timestamp(start_date),
API._timestamp(end_date),
freq=freq).tolist()
if len(dates) == 0:
dates.append(
pd.Interval(left=API._timestamp(start_date),
right=API._timestamp(end_date)))
elif dates[-1].right < API._timestamp(end_date):
dates.append(
pd.Interval(dates[-1].right, API._timestamp(end_date)))
@request_retry(self.ID, retry, retry_wait)
def helper(start, start_date, end_date):
if start_date and end_date:
endpoint = f'/api/v1/{ep}?symbol={symbol}&count={API_MAX}&reverse=false&start={start}&startTime={start_date}&endTime={end_date}'
else:
endpoint = f'/api/v1/{ep}?symbol={symbol}&reverse=true'
header = {}
if self.config.key_id and self.config.key_secret:
header = self._generate_signature("GET", endpoint)
header['Accept'] = 'application/json'
return requests.get('{}{}'.format(self.api, endpoint),
headers=header)
for interval in dates:
start = 0
if interval is not None:
end = interval.right
end -= pd.Timedelta(nanoseconds=1)
start_date = str(interval.left).replace(" ", "T") + "Z"
end_date = str(end).replace(" ", "T") + "Z"
while True:
r = helper(start, start_date, end_date)
if r.status_code in {502, 504}:
LOG.warning("%s: %d for URL %s - %s", self.ID,
r.status_code, r.url, r.text)
sleep(retry_wait)
continue
elif r.status_code == 429:
sleep(API_REFRESH)
continue
elif r.status_code != 200:
self._handle_error(r, LOG)
else:
sleep(RATE_LIMIT_SLEEP)
limit = int(r.headers['X-RateLimit-Remaining'])
data = json.loads(r.text, parse_float=decimal.Decimal)
yield data
if len(data) != API_MAX:
break
if limit < 1:
sleep(API_REFRESH)
start += len(data)
def test_datetime_decode_as_string(value):
assert yapic_json.loads(value, parse_date=True) == py_json.loads(value)
bytes_value = value.encode("utf-8")
assert yapic_json.loads(bytes_value,
parse_date=True) == py_json.loads(value)
def test_datetime_decode_invalid(value):
with pytest.raises(ValueError):
yapic_json.loads(value, parse_date=True)
