async def _funding(self, pairs: Iterable): """ { "success": true, "result": [ { "future": "BTC-PERP", "rate": 0.0025, "time": "2019-06-02T08:00:00+00:00" } ] } """ # do not send more than 30 requests per second: doing so will result in HTTP 429 errors rate_limiter = 0.1 # funding rates do not change frequently wait_time = 60 async with aiohttp.ClientSession() as session: while True: for pair in pairs: if '-PERP' not in pair: continue async with session.get(f"https://ftx.com/api/funding_rates?future={pair}") as response: data = await response.text() data = json.loads(data, parse_float=Decimal) last_update = self.funding.get(pair, None) update = str(data['result'][0]['rate']) + str(data['result'][0]['time']) if last_update and last_update == update: continue else: self.funding[pair] = update await self.callback(FUNDING, feed=self.id, symbol=symbol_exchange_to_std(data['result'][0]['future']), rate=data['result'][0]['rate'], timestamp=timestamp_normalize(self.id, data['result'][0]['time'])) await asyncio.sleep(rate_limiter) await asyncio.sleep(wait_time)
async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if 'error' in msg: LOG.error("%s: Error from exchange: %s", self.id, msg) return chan_id = msg[0] if chan_id == 1002: # the ticker channel doesn't have sequence ids # so it should be None, except for the subscription # ack, in which case its 1 seq_id = msg[1] if seq_id is None and self._channel_map[ msg[2][0]] in self.subscription[1002]: await self._ticker(msg[2], timestamp) elif chan_id < 1000: # order book updates - the channel id refers to # the trading pair being updated seq_no = msg[1] if chan_id not in self.seq_no: self.seq_no[chan_id] = seq_no elif self.seq_no[chan_id] + 1 != seq_no and msg[2][0][0] != 'i': LOG.warning( "%s: missing sequence number. Received %d, expected %d", self.id, seq_no, self.seq_no[chan_id] + 1) raise MissingSequenceNumber self.seq_no[chan_id] = seq_no if msg[2][0][0] == 'i': del self.seq_no[chan_id] symbol = self._channel_map[chan_id] if symbol in self._trade_book_symbols: await self._book(msg[2], chan_id, timestamp) elif chan_id == 1010: # heartbeat - ignore pass else: LOG.warning('%s: Invalid message type %s', self.id, msg)
async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float): msg = json.loads(msg, parse_float=Decimal) # Handle REST endpoint messages first if 'openInterest' in msg: return await self._open_interest(msg, timestamp) # Handle account updates from User Data Stream if self.requires_authentication: msg_type = msg.get('e') if msg_type == 'ACCOUNT_UPDATE': await self._account_update(msg, timestamp) elif msg_type == 'ORDER_TRADE_UPDATE': await self._order_update(msg, timestamp) return # Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>} # streamName is of format <symbol>@<channel> pair, _ = msg['stream'].split('@', 1) msg = msg['data'] pair = pair.upper() msg_type = msg.get('e') if msg_type == 'bookTicker': await self._ticker(msg, timestamp) elif msg_type == 'depthUpdate': await self._book(msg, pair, timestamp) elif msg_type == 'aggTrade': await self._trade(msg, timestamp) elif msg_type == 'forceOrder': await self._liquidations(msg, timestamp) elif msg_type == 'markPriceUpdate': await self._funding(msg, timestamp) elif msg['e'] == 'kline': await self._candle(msg, timestamp) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg)
async def message_handler(self, msg: str, timestamp: float): msg = json.loads(msg, parse_float=Decimal) # Combined stream events are wrapped as follows: {"stream":"<streamName>","data":<rawPayload>} # streamName is of format <symbol>@<channel> pair, _ = msg['stream'].split('@', 1) msg = msg['data'] pair = pair.upper() if msg['e'] == 'depthUpdate': await self._book(msg, pair, timestamp) elif msg['e'] == 'aggTrade': await self._trade(msg, timestamp) elif msg['e'] == '24hrTicker': await self._ticker(msg, timestamp) elif msg['e'] == 'forceOrder': await self._liquidations(msg, timestamp) elif msg['e'] == 'markPriceUpdate': await self._funding(msg, timestamp) else: LOG.warning("%s: Unexpected message received: %s", self.id, msg)
async def message_handler(self, msg: str, conn, timestamp: float): msg_dict = json.loads(msg, parse_float=Decimal) # As a first update after subscription, Deribit sends a notification with no data if "testnet" in msg_dict.keys(): LOG.debug("%s: Test response from derbit accepted %s", self.id, msg) elif "ticker" == msg_dict["params"]["channel"].split(".")[0]: await self._ticker(msg_dict, timestamp) elif "trades" == msg_dict["params"]["channel"].split(".")[0]: await self._trade(msg_dict, timestamp) elif "book" == msg_dict["params"]["channel"].split(".")[0]: # checking if we got full book or its update # if it's update there is 'prev_change_id' field if "prev_change_id" not in msg_dict["params"]["data"].keys(): await self._book_snapshot(msg_dict, timestamp) elif "prev_change_id" in msg_dict["params"]["data"].keys(): await self._book_update(msg_dict, timestamp) else: LOG.warning("%s: Invalid message type %s", self.id, msg)
async def _funding(self, pairs): """ { "status": "ok", "data": { "estimated_rate": "0.000100000000000000", "funding_rate": "-0.000362360011416593", "contract_code": "BTC-USD", "symbol": "BTC", "fee_asset": "BTC", "funding_time": "1603872000000", "next_funding_time": "1603900800000" }, "ts": 1603866304635 } """ while True: for pair in pairs: data = await self.http_conn.read(f'https://api.hbdm.com/swap-api/v1/swap_funding_rate?contract_code={pair}') data = json.loads(data, parse_float=Decimal) received = time.time() update = (data['data']['funding_rate'], self.timestamp_normalize(int(data['data']['next_funding_time']))) if pair in self.funding_updates and self.funding_updates[pair] == update: await asyncio.sleep(1) continue self.funding_updates[pair] = update f = Funding( self.id, self.exchange_symbol_to_std_symbol(pair), None, Decimal(data['data']['funding_rate']), self.timestamp_normalize(int(data['data']['next_funding_time'])), self.timestamp_normalize(int(data['data']['funding_time'])), predicted_rate=Decimal(data['data']['estimated_rate']), raw=data ) await self.callback(FUNDING, f, received) await asyncio.sleep(0.1)
async def _snapshot(self, symbol: str): url = f"https://api.kucoin.com/api/v2/market/orderbook/level2?symbol={symbol}" data = await self.http_conn.read(url) data = json.loads(data, parse_float=Decimal) data = data['data'] self.seq_no[symbol] = int(data['sequence']) self.l2_book[symbol] = { BID: sd({ Decimal(price): Decimal(amount) for price, amount in data['bids'] }), ASK: sd({ Decimal(price): Decimal(amount) for price, amount in data['asks'] }) } timestamp = time.time() await self.book_callback(self.l2_book[symbol], L2_BOOK, symbol, True, None, timestamp, timestamp)
def trade_history(self, symbol: str, start=None, end=None): endpoint = f"/orders?product_id={symbol}&status=done" data = self._request("GET", endpoint, auth=True) data = json.loads(data.text, parse_float=Decimal) return [{ 'order_id': order['id'], 'trade_id': order['id'], 'side': BUY if order['side'] == 'buy' else SELL, 'price': Decimal(order['executed_value']) / Decimal(order['filled_size']), 'amount': Decimal(order['filled_size']), 'timestamp': order['done_at'].timestamp(), 'fee_amount': Decimal(order['fill_fees']), 'fee_currency': symbol.split('-')[1] } for order in data]
async def message_handler(self, msg: str, conn, timestamp: float): # unzip message msg = zlib.decompress(msg, 16 + zlib.MAX_WBITS) msg = json.loads(msg, parse_float=Decimal) # Huobi sends a ping evert 5 seconds and will disconnect us if we do not respond to it if 'ping' in msg: await conn.write(json.dumps({'pong': msg['ping']})) elif 'status' in msg and msg['status'] == 'ok': return elif 'ch' in msg: if 'trade' in msg['ch']: await self._trade(msg, timestamp) elif 'depth' in msg['ch']: await self._book(msg, timestamp) elif 'kline' in msg['ch']: _, symbol, _, interval = msg['ch'].split(".") await self._candles(msg, symbol, interval, timestamp) else: LOG.warning("%s: Invalid message type %s", self.id, msg) else: LOG.warning("%s: Invalid message type %s", self.id, msg)
async def _liquidations(self, pairs: list): last_update = {} async with aiohttp.ClientSession() as session: while True: for pair in pairs: if 'SWAP' in pair: instrument_type = 'swap' else: instrument_type = 'futures' for status in (0, 1): end_point = f"{self.api}{instrument_type}/v3/instruments/{pair}/liquidation?status={status}&limit=100" async with session.get(end_point) as response: data = await response.text() data = json.loads(data, parse_float=Decimal) timestamp = time.time() if len(data) == 0 or (len(data) > 0 and last_update.get(pair) == data[0]): continue for entry in data: if entry == last_update.get(pair): break await self.callback(LIQUIDATIONS, feed=self.id, symbol=entry['instrument_id'], side=BUY if entry['type'] == '3' else SELL, leaves_qty=Decimal(entry['size']), price=Decimal(entry['price']), order_id=None, status='filled' if status == 1 else 'unfilled', timestamp=timestamp, receipt_timestamp=timestamp ) last_update[pair] = data[0] await asyncio.sleep(0.1) await asyncio.sleep(60)
async def trades(self, symbol: str, start=None, end=None, retry_count=1, retry_delay=10): symbol = self.std_symbol_to_exchange_symbol(symbol) start, end = self._interval_normalize(start, end) if start: start = int(start * 1000) end = int(end * 1000) while True: endpoint = f"{self.api}get_last_trades_by_instrument?instrument_name={symbol}&include_old=true&count=1000" if start and end: endpoint = f"{self.api}get_last_trades_by_instrument_and_time?&start_timestamp={start}&end_timestamp={end}&instrument_name={symbol}&include_old=true&count=1000" data = await self.http_conn.read(endpoint, retry_count=retry_count, retry_delay=retry_delay) data = json.loads(data, parse_float=Decimal)["result"]["trades"] if data: if data[-1]["timestamp"] == start: LOG.warning( "%s: number of trades exceeds exchange time window, some data will not be retrieved for time %d", self.id, start) start += 1 else: start = data[-1]["timestamp"] orig_data = data data = [self._trade_normalization(x) for x in data] yield data if len(orig_data) < 1000 or not start or not end: break await asyncio.sleep(1 / self.request_limit)
async def message_handler(self, msg: str, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if 'product_id' in msg and 'sequence' in msg and ('full' in self.channels or ('full' in self.config and msg['product_id'] in self.config['full'])): pair = pair_exchange_to_std(msg['product_id']) if msg['sequence'] <= self.seq_no[pair]: return elif ('full' in self.channels or 'full' in self.config) and msg['sequence'] != self.seq_no[pair] + 1: LOG.warning("%s: Missing sequence number detected for %s", self.id, pair) LOG.warning("%s: Requesting book snapshot", self.id) await self._book_snapshot(self.pairs or self.book_pairs) return self.seq_no[pair] = msg['sequence'] if 'type' in msg: if msg['type'] == 'ticker': await self._ticker(msg, timestamp) elif msg['type'] == 'match' or msg['type'] == 'last_match': await self._book_update(msg, timestamp) elif msg['type'] == 'snapshot': await self._pair_level2_snapshot(msg, timestamp) elif msg['type'] == 'l2update': await self._pair_level2_update(msg, timestamp) elif msg['type'] == 'open': await self._open(msg, timestamp) elif msg['type'] == 'done': await self._done(msg, timestamp) elif msg['type'] == 'change': await self._change(msg, timestamp) elif msg['type'] == 'received': pass elif msg['type'] == 'activate': pass elif msg['type'] == 'subscriptions': pass else: LOG.warning("%s: Invalid message type %s", self.id, msg)
async def _request(self, method: str, endpoint: str, auth: bool = False, body=None, retry_count=1, retry_delay=60): api = self.sandbox_api if self.sandbox else self.api header = None if auth: header = self._generate_signature( endpoint, method, body=json.dumps(body) if body else '') if method == "GET": data = await self.http_conn.read(f'{api}{endpoint}', header=header) elif method == 'POST': data = await self.http_conn.write(f'{api}{endpoint}', msg=body, header=header) elif method == 'DELETE': data = await self.http_conn.delete(f'{api}{endpoint}', header=header) return json.loads(data, parse_float=Decimal)
async def _snapshot(self, symbol: str): """ { "id": 2679059670, "asks": [[price, amount], [...], ...], "bids": [[price, amount], [...], ...] } """ url = f'https://api.gateio.ws/api/v4/spot/order_book?currency_pair={symbol}&limit=100&with_id=true' ret = await self.http_conn.read(url) data = json.loads(ret, parse_float=Decimal) symbol = self.exchange_symbol_to_std_symbol(symbol) self.l2_book[symbol] = {} self.last_update_id[symbol] = data['id'] self.l2_book[symbol][BID] = sd({ Decimal(price): Decimal(amount) for price, amount in data['bids'] }) self.l2_book[symbol][ASK] = sd({ Decimal(price): Decimal(amount) for price, amount in data['asks'] })
async def message_handler(self, msg: str, conn: AsyncConnection, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if isinstance(msg, list): hb_skip = False chan_handler = self.handlers.get(msg[0]) if chan_handler is None: if msg[1] == 'hb': hb_skip = True else: LOG.warning('%s: Unregistered channel ID in message %s', conn.uuid, msg) return seq_no = msg[-1] expected = self.seq_no[conn.uuid] + 1 if seq_no != expected: LOG.warning( '%s: missed message (sequence number) received %d, expected %d', conn.uuid, seq_no, expected) raise MissingSequenceNumber self.seq_no[conn.uuid] = seq_no if hb_skip: return await chan_handler(msg, timestamp) elif 'event' not in msg: LOG.warning('%s: Unexpected msg (missing event) from exchange: %s', conn.uuid, msg) elif msg['event'] == 'error': LOG.error('%s: Error from exchange: %s', conn.uuid, msg) elif msg['event'] in ('info', 'conf'): LOG.info('%s: %s from exchange: %s', conn.uuid, msg['event'], msg) elif 'chanId' in msg and 'symbol' in msg: self.register_channel_handler(msg, conn) else: LOG.warning('%s: Unexpected msg from exchange: %s', conn.uuid, msg)
async def _open_interest(self, pairs: Iterable): """ { "success": true, "result": { "volume": 1000.23, "nextFundingRate": 0.00025, "nextFundingTime": "2019-03-29T03:00:00+00:00", "expirationPrice": 3992.1, "predictedExpirationPrice": 3993.6, "strikePrice": 8182.35, "openInterest": 21124.583 } } """ while True: for pair in pairs: # OI only for perp and futures, so check for / in pair name indicating spot if '/' in pair: continue end_point = f"https://ftx.com/api/futures/{pair}/stats" data = await self.http_conn.read(end_point) received = time() data = json.loads(data, parse_float=Decimal) if 'result' in data: oi = data['result']['openInterest'] if oi != self._open_interest_cache.get(pair, None): o = OpenInterest( self.id, self.exchange_symbol_to_std_symbol(pair), oi, None, raw=data) await self.callback(OPEN_INTEREST, o, received) self._open_interest_cache[pair] = oi await asyncio.sleep(1) await asyncio.sleep(60)
async def trades(self, symbol: str, start=None, end=None, retry_count=1, retry_delay=60): symbol = self.std_symbol_to_exchange_symbol(symbol) start, end = self._interval_normalize(start, end) if start and end: start = int(start * 1000) end = int(end * 1000) while True: if start and end: endpoint = f"{self.api}aggTrades?symbol={symbol}&limit=1000&startTime={start}&endTime={end}" else: endpoint = f"{self.api}aggTrades?symbol={symbol}&limit=1000" r = await self.http_conn.read(endpoint, retry_count=retry_count, retry_delay=retry_delay) data = json.loads(r, parse_float=Decimal) if data: if data[-1]['T'] == start: LOG.warning( "%s: number of trades exceeds exchange time window, some data will not be retrieved for time %d", self.id, start) start += 1 else: start = data[-1]['T'] yield [self._trade_normalization(symbol, d) for d in data] if len(data) < 1000 or end is None: break await asyncio.sleep(1 / self.request_limit)
async def _funding(self, pairs): async with aiohttp.ClientSession() as session: while True: for pair in pairs: async with session.get(f'https://api.hbdm.com/swap-api/v1/swap_funding_rate?contract_code={pair}') as response: data = await response.text() data = json.loads(data, parse_float=Decimal) received = time.time() update = (data['data']['funding_rate'], timestamp_normalize(self.id, int(data['data']['next_funding_time']))) if pair in self.funding_updates and self.funding_updates[pair] == update: await asyncio.sleep(1) continue self.funding_updates[pair] = update await self.callback(FUNDING, feed=self.id, symbol=pair, timestamp=timestamp_normalize(self.id, data['ts']), receipt_timestamp=received, rate=Decimal(update[0]), next_funding_time=update[1] ) await asyncio.sleep(0.1)
async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if 'table' in msg: if msg['table'] == 'trade': await self._trade(msg, timestamp) elif msg['table'] == 'order': await self._order(msg, timestamp) elif msg['table'] == 'orderBookL2': await self._book(msg, timestamp) elif msg['table'] == 'funding': await self._funding(msg, timestamp) elif msg['table'] == 'instrument': await self._instrument(msg, timestamp) elif msg['table'] == 'quote': await self._ticker(msg, timestamp) elif msg['table'] == 'liquidation': await self._liquidation(msg, timestamp) else: LOG.warning("%s: Unhandled table=%r in %r", conn.uuid, msg['table'], msg) elif 'info' in msg: LOG.debug("%s: Info message from exchange: %s", conn.uuid, msg) elif 'subscribe' in msg: if not msg['success']: LOG.error("%s: Subscribe failure: %s", conn.uuid, msg) elif 'error' in msg: LOG.error("%s: Error message from exchange: %s", conn.uuid, msg) elif 'request' in msg: if msg['success']: LOG.debug("%s: Success %s", conn.uuid, msg['request'].get('op')) else: LOG.warning("%s: Failure %s", conn.uuid, msg['request']) else: LOG.warning("%s: Unexpected message from exchange: %s", conn.uuid, msg)
async def trades(self, symbol: str, start=None, end=None, retry_count=1, retry_delay=60): symbol = self.std_symbol_to_exchange_symbol(symbol) last = [] start, end = self._interval_normalize(start, end) while True: endpoint = f"{self.api}/markets/{symbol}/trades" if start and end: endpoint = f"{self.api}/markets/{symbol}/trades?start_time={start}&end_time={end}" r = await self.http_conn.read(endpoint, retry_count=retry_count, retry_delay=retry_delay) data = json.loads(r, parse_float=Decimal)['result'] orig_data = list(data) data = self._dedupe(data, last) last = list(orig_data) data = [self._trade_normalization(x, symbol) for x in data] yield data if len(orig_data) < 5000: break end = int(data[-1]['timestamp']) await asyncio.sleep(1 / self.request_limit)
async def message_handler(self, msg: str, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if isinstance(msg, list): if self.channel_map[msg[0]][0] == 'trade': await self._trade(msg, self.channel_map[msg[0]][1], timestamp) elif self.channel_map[msg[0]][0] == 'ticker': await self._ticker(msg, self.channel_map[msg[0]][1], timestamp) elif self.channel_map[msg[0]][0] == 'book': await self._book(msg, self.channel_map[msg[0]][1], timestamp) else: LOG.warning("%s: No mapping for message %s", self.id, msg) else: if msg['event'] == 'heartbeat': return elif msg['event'] == 'systemStatus': return elif msg['event'] == 'subscriptionStatus' and msg[ 'status'] == 'subscribed': self.channel_map[msg['channelID']] = ( msg['subscription']['name'], pair_exchange_to_std(msg['pair'])) else: LOG.warning("%s: Invalid message type %s", self.id, msg)
def main(filename): with open(filename, 'r') as fp: counter = 0 for line in fp.readlines(): counter += 1 if line == "\n": continue if line.startswith("configuration"): continue start = line[:3] if start == 'wss': continue if start == 'htt': _, line = line.split(" -> ") _, line = line.split(": ", 1) if "header: " in line: line = line.split("header:")[0] try: if 'OKCOIN' in filename or 'OKEX' in filename: if line.startswith('b\'') or line.startswith('b"'): line = bytes_string_to_bytes(line) line = zlib.decompress(line, -15).decode() elif 'HUOBI' in filename and 'ws' in filename: line = bytes_string_to_bytes(line) line = zlib.decompress(line, 16 + zlib.MAX_WBITS) elif 'UPBIT' in filename: if line.startswith('b\'') or line.startswith('b"'): line = line.strip()[2:-1] _ = json.loads(line) except Exception: print(f"Failed on line {counter}: ") print(line) raise print(f"Successfully verified {counter} updates")
async def message_handler(self, msg: str, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if 'info' in msg: LOG.info("%s - info message: %s", self.id, msg) elif 'subscribe' in msg: if not msg['success']: LOG.error("%s: subscribe failed: %s", self.id, msg) elif 'error' in msg: LOG.error("%s: Error message from exchange: %s", self.id, msg) else: if msg['table'] == 'trade': await self._trade(msg, timestamp) elif msg['table'] == 'orderBookL2': await self._book(msg, timestamp) elif msg['table'] == 'funding': await self._funding(msg, timestamp) elif msg['table'] == 'instrument': await self._instrument(msg, timestamp) elif msg['table'] == 'quote': await self._ticker(msg, timestamp) elif msg['table'] == 'liquidation': await self._liquidation(msg, timestamp) else: LOG.warning("%s: Unhandled message %s", self.id, msg)
async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg, parse_float=Decimal) if isinstance(msg, list): channel_id = msg[0] if channel_id not in self.channel_map: LOG.warning("%s: Invalid channel id received %d", self.id, channel_id) LOG.warning("%s: message with invalid channel id: %s", self.id, msg) LOG.warning("%s: channel map: %s", self.id, self.channel_map) else: channel, pair = self.channel_map[channel_id] if channel == 'trade': await self._trade(msg, pair, timestamp) elif channel == 'ticker': await self._ticker(msg, pair, timestamp) elif channel == 'book': await self._book(msg, pair, timestamp) else: LOG.warning("%s: No mapping for message %s", self.id, msg) LOG.warning("%s: channel map: %s", self.id, self.channel_map) else: if msg['event'] == 'heartbeat': return elif msg['event'] == 'systemStatus': return elif msg['event'] == 'subscriptionStatus' and msg[ 'status'] == 'subscribed': self.channel_map[msg['channelID']] = ( msg['subscription']['name'], symbol_exchange_to_std(msg['pair'])) else: LOG.warning("%s: Invalid message type %s", self.id, msg)
async def message_handler(self, msg: str, conn, timestamp: float): msg = json.loads(msg) await self._market_info(msg, timestamp)
def test_datetime_decode(value, expected): assert yapic_json.loads(value, parse_date=True) == expected bytes_value = value.encode("utf-8") assert yapic_json.loads(bytes_value, parse_date=True) == expected
async def _get(self, command: str, retry_count, retry_delay, params=''): api = self.api if not self.sandbox else self.sandbox_api resp = await self.http_conn.read(f"{api}{command}{params}", retry_count=retry_count, retry_delay=retry_delay) return json.loads(resp, parse_float=Decimal)
def _get(self, ep, symbol, start_date, end_date, retry, retry_wait, freq='6H'): dates = [None] if start_date: if not end_date: end_date = pd.Timestamp.utcnow() dates = pd.interval_range(API._timestamp(start_date), API._timestamp(end_date), freq=freq).tolist() if len(dates) == 0: dates.append( pd.Interval(left=API._timestamp(start_date), right=API._timestamp(end_date))) elif dates[-1].right < API._timestamp(end_date): dates.append( pd.Interval(dates[-1].right, API._timestamp(end_date))) @request_retry(self.ID, retry, retry_wait) def helper(start, start_date, end_date): if start_date and end_date: endpoint = f'/api/v1/{ep}?symbol={symbol}&count={API_MAX}&reverse=false&start={start}&startTime={start_date}&endTime={end_date}' else: endpoint = f'/api/v1/{ep}?symbol={symbol}&reverse=true' header = {} if self.config.key_id and self.config.key_secret: header = self._generate_signature("GET", endpoint) header['Accept'] = 'application/json' return requests.get('{}{}'.format(self.api, endpoint), headers=header) for interval in dates: start = 0 if interval is not None: end = interval.right end -= pd.Timedelta(nanoseconds=1) start_date = str(interval.left).replace(" ", "T") + "Z" end_date = str(end).replace(" ", "T") + "Z" while True: r = helper(start, start_date, end_date) if r.status_code in {502, 504}: LOG.warning("%s: %d for URL %s - %s", self.ID, r.status_code, r.url, r.text) sleep(retry_wait) continue elif r.status_code == 429: sleep(API_REFRESH) continue elif r.status_code != 200: self._handle_error(r, LOG) else: sleep(RATE_LIMIT_SLEEP) limit = int(r.headers['X-RateLimit-Remaining']) data = json.loads(r.text, parse_float=decimal.Decimal) yield data if len(data) != API_MAX: break if limit < 1: sleep(API_REFRESH) start += len(data)
def test_datetime_decode_as_string(value): assert yapic_json.loads(value, parse_date=True) == py_json.loads(value) bytes_value = value.encode("utf-8") assert yapic_json.loads(bytes_value, parse_date=True) == py_json.loads(value)
def test_datetime_decode_invalid(value): with pytest.raises(ValueError): yapic_json.loads(value, parse_date=True)