def zmq_socket(ws): """ Note: Currently unused This transfers data from ZMQ to a web socket. The idea was to do some pre-processing in Python before handing it to the WebSocket and consume it in the web interface. However from the documentation it's not very clear how the data provided in `EventList` is formatted. """ LOGGER.debug("Entering ZMQ loop") while True: resp = socket.recv_multipart()[-1] msg = Message() msg.ParseFromString(resp) if msg.message_type == Message.CLIENT_EVENTS: events = EventList() events.ParseFromString(msg.content) LOGGER.debug("Received events") LOGGER.debug(events) # Eventually what we want to do here is ws.send(events) # But before we have to unpack and pre-process it. elif msg.message_type == Message.PING_REQUEST: LOGGER.debug("Received ping request") ws.send("ping request") else: LOGGER.warn("Unexpected message type '{}'".format( msg.message_type ))
async def send(self, message_type, message_content, timeout=None): correlation_id = uuid.uuid4().hex self._msg_router.expect_reply(correlation_id) message = Message( correlation_id=correlation_id, content=message_content, message_type=message_type) # Send the message. Backoff and retry in case of an error # We want a short backoff and retry attempt, so use the defaults # of 3 retries with 200ms of backoff backoff = _Backoff(max_retries=3, interval=200, error=SendBackoffTimeoutError()) while True: try: await self._socket.send_multipart( [message.SerializeToString()]) break except asyncio.CancelledError: # pylint: disable=try-except-raise raise except zmq.error.Again as e: await backoff.do_backoff(err_msg=repr(e)) return await self._msg_router.await_reply(correlation_id, timeout=timeout)
def receive(self): """ Receive a message back. Does not parse the message content. """ ident, result = self._loop.run_until_complete(self._receive()) # Deconstruct the message message = Message() message.ParseFromString(result) LOGGER.info("Received %s(%s) from %s", str(to_protobuf_class(message.message_type).__name__), str(message.message_type), str(ident)) return message, ident
async def produce_custom_msg(self, stream, validated_data): batch_id = validated_data['id'] router = stream.router try: result = await router.list_statuses([batch_id]) except Exception as e: LOGGER.warning(f'Error during fetch: {e}') return data = result['data'][0] resp = { 'id': batch_id, 'status': data['status'], } try: error = data['invalid_transactions'][0]['message'] resp['error'] = error except (KeyError, IndexError): pass evt_resp = _create_event_payload(Events.REMME_BATCH_DELTA.value, resp) correlation_id = uuid.uuid4().hex msg = Message(correlation_id=correlation_id, content=evt_resp.SerializeToString(), message_type=Message.CLIENT_EVENTS) await stream.route_msg(msg)
async def subscribe_events(self, web_sock, last_known_block_id=None): # Setup a connection to the validator LOGGER.debug(f"Subscription started") request = ClientEventsSubscribeRequest( subscriptions=self._make_subscriptions(), last_known_block_ids=[last_known_block_id] if last_known_block_id else []).SerializeToString() # Construct the message wrapper correlation_id = generate_random_key( ) # This must be unique for all in-process requests msg = Message(correlation_id=correlation_id, message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=request) # Send the request LOGGER.debug(f"Sending subscription request.") await self._loop.run_in_executor( None, lambda: self._socket.send_multipart([msg.SerializeToString()])) LOGGER.debug(f"Subscription request is sent") msg = await self._event_queue[Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE ].get() LOGGER.info(f"message type {msg.message_type}") # Validate the response type if msg.message_type != Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE: LOGGER.error(f"Skip unexpected message type") return # Parse the response response = ClientEventsSubscribeResponse() response.ParseFromString(msg.content) # Validate the response status if response.status != ClientEventsSubscribeResponse.OK: if response.status == ClientEventsSubscribeResponse.UNKNOWN_BLOCK: raise SocketException( web_sock, Status.UNKNOWN_BLOCK, f"Uknown block in: {last_known_block_id}") raise SocketException( web_sock, 0, "Subscription failed: Couldn't send multipart") LOGGER.debug(f"Successfully subscribed")
def receive(self): """ Receive a message back. Does not parse the message content. """ ident, result = self._loop.run_until_complete( self._receive() ) # Deconstruct the message message = Message() message.ParseFromString(result) print("Received {}({}) from {}".format( to_protobuf_class(message.message_type).__name__, message.message_type, ident )) return message, ident
def receive(self): """ Receive a message back. Does not parse the message content. """ ident, result = self._loop.run_until_complete(self._receive()) if self._ident is None: self._ident = ident LOGGER.debug(result) LOGGER.debug("%s:%s", len(result), binascii.hexlify(result)) # Deconstruct the message message = Message() message.ParseFromString(result) LOGGER.debug("Received %s from %s", str(message.message_type), str(ident)) return message
async def start(self): """Starts receiving messages on the underlying socket and passes them to the message router. """ self._is_running = True while self._is_running: try: zmq_msg = await self._socket.recv_multipart() message = Message() message.ParseFromString(zmq_msg[-1]) await self._msg_router.route_msg(message) except DecodeError as e: LOGGER.warning('Unable to decode: %s', e) except zmq.ZMQError as e: LOGGER.warning('Unable to receive: %s', e) return except asyncio.CancelledError: self._is_running = False
async def collect_msg_to_queue(self): while True: await asyncio.sleep(2) try: resp = await self._loop.run_in_executor( None, lambda: self._socket.recv_multipart(flags=zmq.NOBLOCK)[-1]) except zmq.Again as e: LOGGER.debug("No message received yet") continue # Parse the message wrapper msg = Message() msg.ParseFromString(resp) try: queue = self._event_queue[msg.message_type] except KeyError: LOGGER.error("Unexpected message type") continue await queue.put(msg)
def send(self, message_content, correlation_id=None): """ Convert the message content to a protobuf message, including serialization of the content and insertion of the content type. Optionally include the correlation id in the message. Messages are sent with the name of this class as the sender. """ message = Message( message_type=to_message_type(message_content), content=message_content.SerializeToString(), correlation_id=correlation_id, ) return self._loop.run_until_complete( self._send(self._tp_ident, message))
def send(self, message_type, message_content, correlation_id=None): """ Convert the message content to a protobuf message, including serialization of the content and insertion of the content type. Optionally include the correlation id in the message. Messages are sent with the name of this class as the sender. """ if self._ident is None: raise ValueError("Must receive a message first.") message = Message( message_type=message_type, content=message_content.SerializeToString(), correlation_id=correlation_id, ) return self._loop.run_until_complete(self._send(self._ident, message))
def watch_batch(self, batch_id): # Setup a connection to the validator ctx = zmq.Context() socket = ctx.socket(zmq.DEALER) socket.connect(current_app.config['SAWTOOTH_VALIDATOR_URL']) # Construct the request request = client_batch_submit_pb2.ClientBatchStatusRequest( batch_ids=[batch_id], wait=True).SerializeToString() # Construct the message wrapper correlation_id = batch_id + uuid.uuid4( ).hex # This must be unique for all in-process requests msg = Message(correlation_id=correlation_id, message_type=Message.CLIENT_BATCH_STATUS_REQUEST, content=request) # Send the request socket.send_multipart([msg.SerializeToString()]) # Receive the response resp = socket.recv_multipart()[-1] # Parse the message wrapper msg = Message() msg.ParseFromString(resp) # Validate the response type if msg.message_type != Message.CLIENT_BATCH_STATUS_RESPONSE: current_app.logger.error("Unexpected response message type") return # Parse the response response = client_batch_submit_pb2.ClientBatchStatusResponse() response.ParseFromString(msg.content) # Validate the response status if response.status != client_batch_submit_pb2.ClientBatchSubmitResponse.OK: current_app.logger.error("watch batch status failed: {}".format( response.response_message)) return # Close the connection to the validator socket.close() return client_batch_submit_pb2.ClientBatchStatus.Status.Name( response.batch_statuses[0].status)
def set_up_zmq_subscription(): """ This sends a subscription request to ZMQ. See: https://sawtooth.hyperledger.org/docs/core/ releases/latest/app_developers_guide/zmq_event_subscription.html """ subscription = EventSubscription( event_type="sawtooth/state-delta", filters=[ EventFilter( key="address", match_string="b89bcb.*", filter_type=EventFilter.REGEX_ANY) ]) request = ClientEventsSubscribeRequest( subscriptions=[subscription] ).SerializeToString() correlation_id = "123" # This must be unique for all in-process requests msg = Message( correlation_id=correlation_id, message_type=Message.MessageType.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=request ) socket.send_multipart([msg.SerializeToString()]) resp = socket.recv_multipart()[-1] msg = Message() msg.ParseFromString(resp) if msg.message_type != \ Message.MessageType.CLIENT_EVENTS_SUBSCRIBE_RESPONSE: print("Unexpected message type") exit(1) response = ClientEventsSubscribeResponse() response.ParseFromString(msg.content) if response.status != ClientEventsSubscribeResponse.OK: print("Subscription failed: {}".format(response.response_message)) exit(1) LOGGER.debug("Setting up ZMQ subscription successful")
def send_req_rep(self, request, request_type): message = Message(message_type=request_type, correlation_id=generate_correlation_id(), content=request.SerializeToString()) self.socket.send_multipart( [self.connection_id, message.SerializeToString()], 0) # pylint: disable=unbalanced-tuple-unpacking _, reply_bytes = self.socket.recv_multipart(0) reply = Message() reply.ParseFromString(reply_bytes) self.assertEqual(reply.message_type, Message.CONSENSUS_NOTIFY_ACK) return reply.content
def recv_rep(self, request_type, response, response_type): # pylint: disable=unbalanced-tuple-unpacking connection_id, message_bytes = self.socket.recv_multipart(0) self.connection_id = connection_id message = Message() message.ParseFromString(message_bytes) request = request_type() request.ParseFromString(message.content) reply = Message(message_type=response_type, content=response.SerializeToString(), correlation_id=message.correlation_id) self.socket.send_multipart( [self.connection_id, reply.SerializeToString()], 0) return request
#Send Event Subscription #Connect to validator ctx = zmq.Context() socket = ctx.socket(zmq.DEALER) socket.connect(url) #Construct the request request = ClientEventsSubscribeRequest( subscriptions=[subscription]).SerializeToString() #Construct the message wrapper correlation_id = "123" msg = Message(correlation_id=correlation_id, message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=request) #Send the request socket.send_multipart([msg.SerializeToString()]) ##################################################################### #Receive the response resp = socket.recv_multipart()[-1] #Parse the msg wrapper msg = Message() msg.ParseFromString(resp) #Validate the response type
def subscribe_event(): """ A custom management command to store block information of each block data by event subscription. this command is calling from docker-compose file """ logger.info("event function start..") # Setup a connection to the validator ctx = zmq.Context() socket = ctx.socket(zmq.DEALER) socket.connect(validator_url) # -------------------------------# # Submit the Event Subscription # # -------------------------------# # subscribe both both "block commit" and "state-delta" event block_sub = EventSubscription(event_type='sawtooth/block-commit') delta_sub = EventSubscription( event_type='sawtooth/state-delta', filters=[ EventFilter(key='address', match_string='^{}.*'.format(NAMESPACE), filter_type=EventFilter.REGEX_ANY) ]) # Construct the request request = ClientEventsSubscribeRequest( subscriptions=[block_sub, delta_sub]).SerializeToString() # Construct the message wrapper correlation_id = "123" # This must be unique for all in-process requests msg = Message(correlation_id=correlation_id, message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=request) # Send the request socket.send_multipart([msg.SerializeToString()]) # # -------------------------------# # Receive the response # # -------------------------------# resp = socket.recv_multipart()[-1] # Parse the message wrapper msg = Message() msg.ParseFromString(resp) # Validate the response type if msg.message_type != Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE: logger.error("Unexpected message type") return # Parse the response response = ClientEventsSubscribeResponse() response.ParseFromString(msg.content) # Validate the response status if response.status != ClientEventsSubscribeResponse.OK: return while True: resp = socket.recv_multipart()[-1] # Parse the message wrapper msg = Message() msg.ParseFromString(resp) # Validate the response type if msg.message_type != Message.CLIENT_EVENTS: return # Parse the response events = EventList() events.ParseFromString(msg.content) block_info = {"address": []} event_data = MessageToDict(events) events_list = event_data["events"] for event in events_list: attributes = event["attributes"] for attr in attributes: key = attr["key"] value = attr["value"] if key == 'address': block_info["address"].append(value) else: block_info[key] = value address_list = block_info["address"] for address in address_list: BlockInfo.objects.create( block_num=block_info["block_num"], block_id=block_info["block_id"], state_root_hash=block_info["state_root_hash"], previous_block_id=block_info["previous_block_id"], address=address, ) logger.info("blockinfo subscription created..")
def subscribe(self, event_name: str, is_write_to_file=False, file_name='certificate.pem'): subscription = EventSubscription(event_type="ca_1/{}".format(event_name)) # Setup a connection to the validator ctx = zmq.Context() socket = ctx.socket(zmq.DEALER) socket.connect('tcp://127.0.0.1:4004') # Construct the request request = ClientEventsSubscribeRequest( subscriptions=[subscription]).SerializeToString() # Construct the message wrapper correlation_id = "123" # This must be unique for all in-process requests msg = Message( correlation_id=correlation_id, message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=request) # Send the request socket.send_multipart([msg.SerializeToString()]) # Receive the response resp = socket.recv_multipart()[-1] # Parse the message wrapper msg = Message() msg.ParseFromString(resp) # Validate the response type if msg.message_type != Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE: print("Unexpected message type") return # Parse the response response = ClientEventsSubscribeResponse() response.ParseFromString(msg.content) # Validate the response status if response.status != ClientEventsSubscribeResponse.OK: print("Subscription failed: {}".format(response.response_message)) return resp = socket.recv_multipart()[-1] # Parse the message wrapper msg = Message() msg.ParseFromString(resp) # Validate the response type if msg.message_type != Message.CLIENT_EVENTS: print("Unexpected message type") return # Parse the response events = EventList() events.ParseFromString(msg.content) for event in events.events: if event.data is not None: if is_write_to_file: write_to_file(file_name, event.data) else: print(event.data) # Construct the request request = ClientEventsUnsubscribeRequest().SerializeToString() # Construct the message wrapper correlation_id = "124" # This must be unique for all in-process requests msg = Message( correlation_id=correlation_id, message_type=Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, content=request) # Send the request socket.send_multipart([msg.SerializeToString()]) # Receive the response resp = socket.recv_multipart()[-1] # Parse the message wrapper msg = Message() msg.ParseFromString(resp) # Validate the response type if msg.message_type != Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE: print("Unexpected message type") # Parse the response response = ClientEventsUnsubscribeResponse() response.ParseFromString(msg.content) # Validate the response status if response.status != ClientEventsUnsubscribeResponse.OK: print("Unsubscription failed: {}".format(response.response_message)) # Close the connection to the validator socket.close()