def process_stream_data(received_stream_data_json, stream_buffer_name="False"): # # START HERE! # # `received_stream_data_json` contains one record of raw data from the stream # print it and you see the data like its given from Binance, its hard to work with them, because keys of # parameters are changing from stream to stream and they are not self explaining. # # So if you want, you can use the class `UnicornFy`, it converts the json to a dict and prepares the values. # `depth5` for example doesnt include the symbol, but the unicornfied set includes them, because the class # extracts it from the channel name, makes it upper size and adds it to the returned values.. just print both # to see the difference. # Github: https://github.com/LUCIT-Systems-and-Development/unicorn-fy # PyPI: https://pypi.org/project/unicorn-fy/ exchange = "binance.com" if exchange == "binance.com" or exchange == "binance.com-testnet": unicorn_fied_stream_data = UnicornFy.binance_com_websocket(received_stream_data_json) elif exchange == "binance.com-futures" or exchange == "binance.com-futures-testnet": unicorn_fied_stream_data = UnicornFy.binance_com_futures_websocket(received_stream_data_json) elif exchange == "binance.com-margin" or exchange == "binance.com-margin-testnet": unicorn_fied_stream_data = UnicornFy.binance_com_margin_websocket(received_stream_data_json) elif exchange == "binance.com-isolated_margin" or exchange == "binance.com-isolated_margin-testnet": unicorn_fied_stream_data = UnicornFy.binance_com_margin_websocket(received_stream_data_json) elif exchange == "binance.je": unicorn_fied_stream_data = UnicornFy.binance_je_websocket(received_stream_data_json) elif exchange == "binance.us": unicorn_fied_stream_data = UnicornFy.binance_us_websocket(received_stream_data_json) else: logging.error("Not a valid exchange: " + str(exchange)) # Now you can call different methods for different `channels`, here called `event_types`. # Its up to you if you call the methods in the bottom of this file or to call other classes which do what # ever you want to be done. try: if unicorn_fied_stream_data['event_type'] == "aggTrade": BinanceWebSocketApiProcessStreams.aggtrade(unicorn_fied_stream_data) elif unicorn_fied_stream_data['event_type'] == "trade": BinanceWebSocketApiProcessStreams.trade(unicorn_fied_stream_data) elif unicorn_fied_stream_data['event_type'] == "kline": BinanceWebSocketApiProcessStreams.kline(unicorn_fied_stream_data) elif unicorn_fied_stream_data['event_type'] == "24hrMiniTicker": BinanceWebSocketApiProcessStreams.miniticker(unicorn_fied_stream_data) elif unicorn_fied_stream_data['event_type'] == "24hrTicker": BinanceWebSocketApiProcessStreams.ticker(unicorn_fied_stream_data) elif unicorn_fied_stream_data['event_type'] == "depth": BinanceWebSocketApiProcessStreams.miniticker(unicorn_fied_stream_data) else: BinanceWebSocketApiProcessStreams.anything_else(unicorn_fied_stream_data) except KeyError: BinanceWebSocketApiProcessStreams.anything_else(unicorn_fied_stream_data) except TypeError: pass
class TestBinanceGeneric(unittest.TestCase): def setUp(self): print("\n\rstarting unittests") self.unicorn_fy = UnicornFy() self.unicorn_fy_version = str(self.unicorn_fy.get_version()) self.unicorn_fy.get_latest_release_info() self.unicorn_fy.get_latest_version() def test_set_to_false_if_not_exist(self): value = {'key': '', 'blub': 2} self.assertEqual(str(self.unicorn_fy.set_to_false_if_not_exist(value, "invalid")), "{'key': '', 'blub': 2, 'invalid': False}") def test_is_json(self): self.assertFalse(self.unicorn_fy.is_json(False)) def test_result(self): data = '{"result":null,"id":2}' asserted_result = "{'result': None, 'id': 2, 'unicorn_fied': ['binance.com', '" + self.unicorn_fy_version + "']}" self.assertEqual(str(self.unicorn_fy.binance_com_websocket(data)), asserted_result) def test_error(self): data = '{"error":"blahblah"}' asserted_result = "{'error': 'blahblah', 'unicorn_fied': ['binance.com', '" + self.unicorn_fy_version + "']}" self.assertEqual(str(self.unicorn_fy.binance_com_websocket(data)), asserted_result) def test_template(self): data = '' asserted_result = "" self.assertEqual(str(self.unicorn_fy.binance_com_margin_websocket(data)), asserted_result) def tearDown(self): del self.unicorn_fy
class TestBinanceComMarginWebsocket(unittest.TestCase): def setUp(self): self.unicorn_fy = UnicornFy() self.unicorn_fy_version = str(self.unicorn_fy.get_version()) def test_template(self): data = '' asserted_result = "" self.assertEqual(str(self.unicorn_fy.binance_com_margin_websocket(data)), asserted_result) def tearDown(self): del self.unicorn_fy
class BinanceWebSocketApiSocket(object): def __init__(self, manager, stream_id, channels, markets): self.manager = manager self.stream_id = stream_id self.channels = channels self.markets = markets self.socket_id = uuid.uuid4() self.manager.stream_list[ self.stream_id]['recent_socket_id'] = self.socket_id self.symbols = self.manager.stream_list[self.stream_id]['symbols'] self.output = self.manager.stream_list[self.stream_id]['output'] self.unicorn_fy = UnicornFy() self.exchange = manager.get_exchange() async def start_socket(self): logging.info( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, {str(self.channels)}, " f"{str(self.markets)}) socket_id={str(self.socket_id)} recent_socket_id={str(self.socket_id)}" ) async with BinanceWebSocketApiConnection( self.manager, self.stream_id, self.socket_id, self.channels, self.markets, symbols=self.symbols) as websocket: while True: if self.manager.is_stop_request(self.stream_id): self.manager.stream_is_stopping(self.stream_id) await websocket.close() sys.exit(0) elif self.manager.is_stop_as_crash_request(self.stream_id): await websocket.close() sys.exit(1) try: if self.manager.stream_list[self.stream_id][ 'recent_socket_id'] != self.socket_id: logging.error( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, " f"{str(self.channels)}, {str(self.markets)} socket_id={str(self.socket_id)} " f"recent_socket_id={str(self.socket_id)} - Sending payload - exit because its " f"not the recent socket id! stream_id={str(self.stream_id)}, recent_socket_id=" f"{str(self.manager.stream_list[self.stream_id]['recent_socket_id'])}" ) sys.exit(0) except KeyError: sys.exit(1) while self.manager.stream_list[self.stream_id]['payload']: logging.info( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, " f"{str(self.channels)}, {str(self.markets)} socket_id={str(self.socket_id)} " f"recent_socket_id={str(self.socket_id)} - Sending payload started ..." ) if self.manager.stream_list[self.stream_id][ 'recent_socket_id'] != self.socket_id: logging.error( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, " f"{str(self.channels)}, {str(self.markets)} socket_id={str(self.socket_id)} " f"recent_socket_id={str(self.socket_id)} - Sending payload - exit because its " f"not the recent socket id! stream_id={str(self.stream_id)}, recent_socket_id=" f"{str(self.manager.stream_list[self.stream_id]['recent_socket_id'])}" ) sys.exit(0) payload = self.manager.stream_list[ self.stream_id]['payload'].pop(0) logging.info( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, " f"{str(self.channels)}, {str(self.markets)} - Sending payload: {str(payload)}" ) await websocket.send( json.dumps(payload, ensure_ascii=False)) # To avoid a ban we respect the limits of binance: # https://github.com/binance-exchange/binance-official-api-docs/blob/5fccfd572db2f530e25e302c02be5dec12759cf9/CHANGELOG.md#2020-04-23 # Limit: max 5 messages per second inclusive pings/pong max_subscriptions_per_second = self.manager.max_send_messages_per_second - \ self.manager.max_send_messages_per_second_reserve idle_time = 1 / max_subscriptions_per_second time.sleep(idle_time) try: received_stream_data_json = await websocket.receive() if received_stream_data_json is not None: if self.output == "UnicornFy": if self.exchange == "binance.com": received_stream_data = self.unicorn_fy.binance_com_websocket( received_stream_data_json) elif self.exchange == "binance.com-testnet": received_stream_data = self.unicorn_fy.binance_com_websocket( received_stream_data_json) elif self.exchange == "binance.com-margin": received_stream_data = self.unicorn_fy.binance_com_margin_websocket( received_stream_data_json) elif self.exchange == "binance.com-margin-testnet": received_stream_data = self.unicorn_fy.binance_com_margin_websocket( received_stream_data_json) elif self.exchange == "binance.com-isolated_margin": received_stream_data = self.unicorn_fy.binance_com_isolated_margin_websocket( received_stream_data_json) elif self.exchange == "binance.com-isolated_margin-testnet": received_stream_data = self.unicorn_fy.binance_com_isolated_margin_websocket( received_stream_data_json) elif self.exchange == "binance.com-futures": received_stream_data = self.unicorn_fy.binance_com_futures_websocket( received_stream_data_json) elif self.exchange == "binance.com-futures-testnet": received_stream_data = self.unicorn_fy.binance_com_futures_websocket( received_stream_data_json) elif self.exchange == "binance.je": received_stream_data = self.unicorn_fy.binance_je_websocket( received_stream_data_json) elif self.exchange == "binance.us": received_stream_data = self.unicorn_fy.binance_us_websocket( received_stream_data_json) elif self.exchange == "jex.com": received_stream_data = self.unicorn_fy.jex_com_websocket( received_stream_data_json) elif self.exchange == "binance.org": received_stream_data = self.unicorn_fy.binance_org_websocket( received_stream_data_json) elif self.exchange == "binance.org-testnet": received_stream_data = self.unicorn_fy.binance_org_websocket( received_stream_data_json) else: received_stream_data = received_stream_data_json elif self.output == "dict": received_stream_data = json.loads( received_stream_data_json) else: received_stream_data = received_stream_data_json try: stream_buffer_name = self.manager.stream_list[ self.stream_id]['stream_buffer_name'] except KeyError: stream_buffer_name = False self.manager.process_stream_data( received_stream_data, stream_buffer_name=stream_buffer_name) if "error" in received_stream_data_json: logging.error( "BinanceWebSocketApiSocket.start_socket(" + str(self.stream_id) + ") " "- Received error message: " + str(received_stream_data_json)) self.manager.add_to_ringbuffer_error( received_stream_data_json) elif "result" in received_stream_data_json: logging.info( "BinanceWebSocketApiSocket.start_socket(" + str(self.stream_id) + ") " "- Received result message: " + str(received_stream_data_json)) self.manager.add_to_ringbuffer_result( received_stream_data_json) except websockets.exceptions.ConnectionClosed as error_msg: logging.critical( "BinanceWebSocketApiSocket.start_socket(" + str(self.stream_id) + ", " + str(self.channels) + ", " + str(self.markets) + ") - Exception ConnectionClosed " "- error_msg: " + str(error_msg)) if "WebSocket connection is closed: code = 1008" in str( error_msg): websocket.close() self.manager.stream_is_crashing( self.stream_id, error_msg) self.manager.set_restart_request(self.stream_id) sys.exit(1) elif "WebSocket connection is closed: code = 1006" in str( error_msg): self.manager.stream_is_crashing( self.stream_id, error_msg) self.manager.set_restart_request(self.stream_id) sys.exit(1) else: self.manager.stream_is_crashing( self.stream_id, str(error_msg)) self.manager.set_restart_request(self.stream_id) sys.exit(1) except AttributeError as error_msg: logging.error("BinanceWebSocketApiSocket.start_socket(" + str(self.stream_id) + ", " + str(self.channels) + ", " + str(self.markets) + ") - Exception AttributeError - " "error_msg: " + str(error_msg)) self.manager.stream_is_crashing(self.stream_id, str(error_msg)) self.manager.set_restart_request(self.stream_id) sys.exit(1)
class BinanceWebSocketApiSocket(object): def __init__(self, manager, stream_id, channels, markets): self.manager = manager self.stream_id = stream_id self.channels = channels self.markets = markets self.socket_id = uuid.uuid4() self.manager.stream_list[ self.stream_id]['recent_socket_id'] = self.socket_id self.symbols = self.manager.stream_list[self.stream_id]['symbols'] self.output = self.manager.stream_list[self.stream_id]['output'] self.unicorn_fy = UnicornFy() self.exchange = manager.get_exchange() async def start_socket(self): logger.info( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, {str(self.channels)}, " f"{str(self.markets)}) socket_id={str(self.socket_id)} recent_socket_id={str(self.socket_id)}" ) try: async with BinanceWebSocketApiConnection( self.manager, self.stream_id, self.socket_id, self.channels, self.markets, symbols=self.symbols) as websocket: self.manager.socket_is_ready[self.stream_id] = True while True: if self.manager.is_stop_request(self.stream_id): self.manager.stream_is_stopping(self.stream_id) await websocket.close() sys.exit(0) elif self.manager.is_stop_as_crash_request(self.stream_id): await websocket.close() sys.exit(1) try: if self.manager.stream_list[self.stream_id][ 'recent_socket_id'] != self.socket_id: logger.error( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, " f"{str(self.channels)}, {str(self.markets)} socket_id={str(self.socket_id)} " f"recent_socket_id={str(self.socket_id)} - Sending payload - exit because its " f"not the recent socket id! stream_id={str(self.stream_id)}, recent_socket_id=" f"{str(self.manager.stream_list[self.stream_id]['recent_socket_id'])}" ) sys.exit(0) except KeyError: sys.exit(1) while self.manager.stream_list[self.stream_id]['payload']: logger.info( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, " f"{str(self.channels)}, {str(self.markets)} socket_id={str(self.socket_id)} " f"recent_socket_id={str(self.socket_id)} - Sending payload started ..." ) if self.manager.stream_list[self.stream_id][ 'recent_socket_id'] != self.socket_id: logger.error( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, " f"{str(self.channels)}, {str(self.markets)} socket_id={str(self.socket_id)} " f"recent_socket_id={str(self.socket_id)} - Sending payload - exit because its " f"not the recent socket id! stream_id={str(self.stream_id)}, recent_socket_id=" f"{str(self.manager.stream_list[self.stream_id]['recent_socket_id'])}" ) sys.exit(0) payload = self.manager.stream_list[ self.stream_id]['payload'].pop(0) logger.info( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, " f"{str(self.channels)}, {str(self.markets)} - Sending payload: {str(payload)}" ) await websocket.send( json.dumps(payload, ensure_ascii=False)) # To avoid a ban we respect the limits of binance: # https://github.com/binance-exchange/binance-official-api-docs/blob/5fccfd572db2f530e25e302c02be5dec12759cf9/CHANGELOG.md#2020-04-23 # Limit: max 5 messages per second inclusive pings/pong max_subscriptions_per_second = self.manager.max_send_messages_per_second - \ self.manager.max_send_messages_per_second_reserve idle_time = 1 / max_subscriptions_per_second await asyncio.sleep(idle_time) try: try: received_stream_data_json = await websocket.receive( ) except asyncio.TimeoutError: # Timeout from `asyncio.wait_for()` which we use to keep the loop running even if we dont # receive new records via websocket. logger.debug( f"BinanceWebSocketApiSocket.start_socket({str(self.stream_id)}, " f"{str(self.channels)}, {str(self.markets)} - Received inner " f"asyncio.TimeoutError") continue if received_stream_data_json is not None: if self.output == "UnicornFy": if self.exchange == "binance.com": received_stream_data = self.unicorn_fy.binance_com_websocket( received_stream_data_json) elif self.exchange == "binance.com-testnet": received_stream_data = self.unicorn_fy.binance_com_websocket( received_stream_data_json) elif self.exchange == "binance.com-margin": received_stream_data = self.unicorn_fy.binance_com_margin_websocket( received_stream_data_json) elif self.exchange == "binance.com-margin-testnet": received_stream_data = self.unicorn_fy.binance_com_margin_websocket( received_stream_data_json) elif self.exchange == "binance.com-isolated_margin": received_stream_data = self.unicorn_fy.binance_com_isolated_margin_websocket( received_stream_data_json) elif self.exchange == "binance.com-isolated_margin-testnet": received_stream_data = self.unicorn_fy.binance_com_isolated_margin_websocket( received_stream_data_json) elif self.exchange == "binance.com-futures": received_stream_data = self.unicorn_fy.binance_com_futures_websocket( received_stream_data_json) elif self.exchange == "binance.com-futures-testnet": received_stream_data = self.unicorn_fy.binance_com_futures_websocket( received_stream_data_json) elif self.exchange == "binance.com-coin-futures" or self.exchange == "binance.com-coin_futures": received_stream_data = self.unicorn_fy.binance_com_coin_futures_websocket( received_stream_data_json) elif self.exchange == "binance.je": received_stream_data = self.unicorn_fy.binance_je_websocket( received_stream_data_json) elif self.exchange == "binance.us": received_stream_data = self.unicorn_fy.binance_us_websocket( received_stream_data_json) elif self.exchange == "trbinance.com": received_stream_data = self.unicorn_fy.trbinance_com_websocket( received_stream_data_json) elif self.exchange == "jex.com": received_stream_data = self.unicorn_fy.jex_com_websocket( received_stream_data_json) elif self.exchange == "binance.org": received_stream_data = self.unicorn_fy.binance_org_websocket( received_stream_data_json) elif self.exchange == "binance.org-testnet": received_stream_data = self.unicorn_fy.binance_org_websocket( received_stream_data_json) else: received_stream_data = received_stream_data_json elif self.output == "dict": received_stream_data = json.loads( received_stream_data_json) else: received_stream_data = received_stream_data_json try: stream_buffer_name = self.manager.stream_list[ self.stream_id]['stream_buffer_name'] except KeyError: stream_buffer_name = False if stream_buffer_name: # if create_stream() got a stram_buffer_name -> use it self.manager.add_to_stream_buffer( received_stream_data, stream_buffer_name=stream_buffer_name) elif self.manager.specific_process_stream_data[ self.stream_id] is not None: # if create_stream() got a callback function -> use it self.manager.specific_process_stream_data[ self.stream_id](received_stream_data) else: # Use the default process_stream_data function provided to/by the manager class self.manager.process_stream_data( received_stream_data, stream_buffer_name=stream_buffer_name) if "error" in received_stream_data_json: logger.error( "BinanceWebSocketApiSocket.start_socket(" + str(self.stream_id) + ") " "- Received error message: " + str(received_stream_data_json)) self.manager.add_to_ringbuffer_error( received_stream_data_json) elif "result" in received_stream_data_json: logger.info( "BinanceWebSocketApiSocket.start_socket(" + str(self.stream_id) + ") " "- Received result message: " + str(received_stream_data_json)) self.manager.add_to_ringbuffer_result( received_stream_data_json) else: if self.manager.stream_list[self.stream_id][ 'last_received_data_record'] is None: self.manager.process_stream_signals( "FIRST_RECEIVED_DATA", self.stream_id, received_stream_data) self.manager.stream_list[self.stream_id][ 'last_received_data_record'] = received_stream_data except websockets.exceptions.ConnectionClosed as error_msg: logger.critical( "BinanceWebSocketApiSocket.start_socket(" + str(self.stream_id) + ", " + str(self.channels) + ", " + str(self.markets) + ") - Exception ConnectionClosed " "- error_msg: " + str(error_msg)) if "WebSocket connection is closed: code = 1008" in str( error_msg): websocket.close() self.manager.stream_is_crashing( self.stream_id, error_msg) self.manager.set_restart_request(self.stream_id) sys.exit(1) elif "WebSocket connection is closed: code = 1006" in str( error_msg): self.manager.stream_is_crashing( self.stream_id, error_msg) self.manager.set_restart_request(self.stream_id) sys.exit(1) else: self.manager.stream_is_crashing( self.stream_id, str(error_msg)) self.manager.set_restart_request(self.stream_id) sys.exit(1) except AttributeError as error_msg: logger.error( "BinanceWebSocketApiSocket.start_socket(" + str(self.stream_id) + ", " + str(self.channels) + ", " + str(self.markets) + ") - Exception AttributeError - " "error_msg: " + str(error_msg)) self.manager.stream_is_crashing( self.stream_id, str(error_msg)) self.manager.set_restart_request(self.stream_id) sys.exit(1) except asyncio.TimeoutError as error_msg: # Catching https://github.com/LUCIT-Systems-and-Development/unicorn-binance-websocket-api/issues/221 self.manager.stream_is_crashing(self.stream_id, error_msg) self.manager.set_restart_request(self.stream_id) sys.exit(1)