def zmq_socket(ws):
"""
Note: Currently unused
This transfers data from ZMQ to a web socket.
The idea was to do some pre-processing in Python
before handing it to the WebSocket and consume it
in the web interface.
However from the documentation it's not very clear
how the data provided in `EventList` is formatted.
"""
LOGGER.debug("Entering ZMQ loop")
while True:
resp = socket.recv_multipart()[-1]
msg = Message()
msg.ParseFromString(resp)
if msg.message_type == Message.CLIENT_EVENTS:
events = EventList()
events.ParseFromString(msg.content)
LOGGER.debug("Received events")
LOGGER.debug(events)
# Eventually what we want to do here is ws.send(events)
# But before we have to unpack and pre-process it.
elif msg.message_type == Message.PING_REQUEST:
LOGGER.debug("Received ping request")
ws.send("ping request")
else:
LOGGER.warn("Unexpected message type '{}'".format(
msg.message_type
))
async def send(self, message_type, message_content, timeout=None):
correlation_id = uuid.uuid4().hex
self._msg_router.expect_reply(correlation_id)
message = Message(
correlation_id=correlation_id,
content=message_content,
message_type=message_type)
# Send the message. Backoff and retry in case of an error
# We want a short backoff and retry attempt, so use the defaults
# of 3 retries with 200ms of backoff
backoff = _Backoff(max_retries=3,
interval=200,
error=SendBackoffTimeoutError())
while True:
try:
await self._socket.send_multipart(
[message.SerializeToString()])
break
except asyncio.CancelledError: # pylint: disable=try-except-raise
raise
except zmq.error.Again as e:
await backoff.do_backoff(err_msg=repr(e))
return await self._msg_router.await_reply(correlation_id,
timeout=timeout)
def receive(self):
"""
Receive a message back. Does not parse the message content.
"""
ident, result = self._loop.run_until_complete(self._receive())
# Deconstruct the message
message = Message()
message.ParseFromString(result)
LOGGER.info("Received %s(%s) from %s",
str(to_protobuf_class(message.message_type).__name__),
str(message.message_type), str(ident))
return message, ident
async def produce_custom_msg(self, stream, validated_data):
batch_id = validated_data['id']
router = stream.router
try:
result = await router.list_statuses([batch_id])
except Exception as e:
LOGGER.warning(f'Error during fetch: {e}')
return
data = result['data'][0]
resp = {
'id': batch_id,
'status': data['status'],
}
try:
error = data['invalid_transactions'][0]['message']
resp['error'] = error
except (KeyError, IndexError):
pass
evt_resp = _create_event_payload(Events.REMME_BATCH_DELTA.value, resp)
correlation_id = uuid.uuid4().hex
msg = Message(correlation_id=correlation_id,
content=evt_resp.SerializeToString(),
message_type=Message.CLIENT_EVENTS)
await stream.route_msg(msg)
async def subscribe_events(self, web_sock, last_known_block_id=None):
# Setup a connection to the validator
LOGGER.debug(f"Subscription started")
request = ClientEventsSubscribeRequest(
subscriptions=self._make_subscriptions(),
last_known_block_ids=[last_known_block_id]
if last_known_block_id else []).SerializeToString()
# Construct the message wrapper
correlation_id = generate_random_key(
) # This must be unique for all in-process requests
msg = Message(correlation_id=correlation_id,
message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST,
content=request)
# Send the request
LOGGER.debug(f"Sending subscription request.")
await self._loop.run_in_executor(
None,
lambda: self._socket.send_multipart([msg.SerializeToString()]))
LOGGER.debug(f"Subscription request is sent")
msg = await self._event_queue[Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE
].get()
LOGGER.info(f"message type {msg.message_type}")
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE:
LOGGER.error(f"Skip unexpected message type")
return
# Parse the response
response = ClientEventsSubscribeResponse()
response.ParseFromString(msg.content)
# Validate the response status
if response.status != ClientEventsSubscribeResponse.OK:
if response.status == ClientEventsSubscribeResponse.UNKNOWN_BLOCK:
raise SocketException(
web_sock, Status.UNKNOWN_BLOCK,
f"Uknown block in: {last_known_block_id}")
raise SocketException(
web_sock, 0, "Subscription failed: Couldn't send multipart")
LOGGER.debug(f"Successfully subscribed")
def receive(self):
"""
Receive a message back. Does not parse the message content.
"""
ident, result = self._loop.run_until_complete(
self._receive()
)
# Deconstruct the message
message = Message()
message.ParseFromString(result)
print("Received {}({}) from {}".format(
to_protobuf_class(message.message_type).__name__,
message.message_type,
ident
))
return message, ident
def receive(self):
"""
Receive a message back. Does not parse the message content.
"""
ident, result = self._loop.run_until_complete(self._receive())
if self._ident is None:
self._ident = ident
LOGGER.debug(result)
LOGGER.debug("%s:%s", len(result), binascii.hexlify(result))
# Deconstruct the message
message = Message()
message.ParseFromString(result)
LOGGER.debug("Received %s from %s", str(message.message_type),
str(ident))
return message
async def start(self):
"""Starts receiving messages on the underlying socket and passes them
to the message router.
"""
self._is_running = True
while self._is_running:
try:
zmq_msg = await self._socket.recv_multipart()
message = Message()
message.ParseFromString(zmq_msg[-1])
await self._msg_router.route_msg(message)
except DecodeError as e:
LOGGER.warning('Unable to decode: %s', e)
except zmq.ZMQError as e:
LOGGER.warning('Unable to receive: %s', e)
return
except asyncio.CancelledError:
self._is_running = False
async def collect_msg_to_queue(self):
while True:
await asyncio.sleep(2)
try:
resp = await self._loop.run_in_executor(
None,
lambda: self._socket.recv_multipart(flags=zmq.NOBLOCK)[-1])
except zmq.Again as e:
LOGGER.debug("No message received yet")
continue
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
try:
queue = self._event_queue[msg.message_type]
except KeyError:
LOGGER.error("Unexpected message type")
continue
await queue.put(msg)
def send(self, message_content, correlation_id=None):
"""
Convert the message content to a protobuf message, including
serialization of the content and insertion of the content type.
Optionally include the correlation id in the message. Messages
are sent with the name of this class as the sender.
"""
message = Message(
message_type=to_message_type(message_content),
content=message_content.SerializeToString(),
correlation_id=correlation_id,
)
return self._loop.run_until_complete(
self._send(self._tp_ident, message))
def send(self, message_type, message_content, correlation_id=None):
"""
Convert the message content to a protobuf message, including
serialization of the content and insertion of the content type.
Optionally include the correlation id in the message. Messages
are sent with the name of this class as the sender.
"""
if self._ident is None:
raise ValueError("Must receive a message first.")
message = Message(
message_type=message_type,
content=message_content.SerializeToString(),
correlation_id=correlation_id,
)
return self._loop.run_until_complete(self._send(self._ident, message))
def watch_batch(self, batch_id):
# Setup a connection to the validator
ctx = zmq.Context()
socket = ctx.socket(zmq.DEALER)
socket.connect(current_app.config['SAWTOOTH_VALIDATOR_URL'])
# Construct the request
request = client_batch_submit_pb2.ClientBatchStatusRequest(
batch_ids=[batch_id], wait=True).SerializeToString()
# Construct the message wrapper
correlation_id = batch_id + uuid.uuid4(
).hex # This must be unique for all in-process requests
msg = Message(correlation_id=correlation_id,
message_type=Message.CLIENT_BATCH_STATUS_REQUEST,
content=request)
# Send the request
socket.send_multipart([msg.SerializeToString()])
# Receive the response
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_BATCH_STATUS_RESPONSE:
current_app.logger.error("Unexpected response message type")
return
# Parse the response
response = client_batch_submit_pb2.ClientBatchStatusResponse()
response.ParseFromString(msg.content)
# Validate the response status
if response.status != client_batch_submit_pb2.ClientBatchSubmitResponse.OK:
current_app.logger.error("watch batch status failed: {}".format(
response.response_message))
return
# Close the connection to the validator
socket.close()
return client_batch_submit_pb2.ClientBatchStatus.Status.Name(
response.batch_statuses[0].status)
def set_up_zmq_subscription():
"""
This sends a subscription request to ZMQ.
See: https://sawtooth.hyperledger.org/docs/core/
releases/latest/app_developers_guide/zmq_event_subscription.html
"""
subscription = EventSubscription(
event_type="sawtooth/state-delta",
filters=[
EventFilter(
key="address",
match_string="b89bcb.*",
filter_type=EventFilter.REGEX_ANY)
])
request = ClientEventsSubscribeRequest(
subscriptions=[subscription]
).SerializeToString()
correlation_id = "123" # This must be unique for all in-process requests
msg = Message(
correlation_id=correlation_id,
message_type=Message.MessageType.CLIENT_EVENTS_SUBSCRIBE_REQUEST,
content=request
)
socket.send_multipart([msg.SerializeToString()])
resp = socket.recv_multipart()[-1]
msg = Message()
msg.ParseFromString(resp)
if msg.message_type != \
Message.MessageType.CLIENT_EVENTS_SUBSCRIBE_RESPONSE:
print("Unexpected message type")
exit(1)
response = ClientEventsSubscribeResponse()
response.ParseFromString(msg.content)
if response.status != ClientEventsSubscribeResponse.OK:
print("Subscription failed: {}".format(response.response_message))
exit(1)
LOGGER.debug("Setting up ZMQ subscription successful")
def send_req_rep(self, request, request_type):
message = Message(message_type=request_type,
correlation_id=generate_correlation_id(),
content=request.SerializeToString())
self.socket.send_multipart(
[self.connection_id,
message.SerializeToString()], 0)
# pylint: disable=unbalanced-tuple-unpacking
_, reply_bytes = self.socket.recv_multipart(0)
reply = Message()
reply.ParseFromString(reply_bytes)
self.assertEqual(reply.message_type, Message.CONSENSUS_NOTIFY_ACK)
return reply.content
def recv_rep(self, request_type, response, response_type):
# pylint: disable=unbalanced-tuple-unpacking
connection_id, message_bytes = self.socket.recv_multipart(0)
self.connection_id = connection_id
message = Message()
message.ParseFromString(message_bytes)
request = request_type()
request.ParseFromString(message.content)
reply = Message(message_type=response_type,
content=response.SerializeToString(),
correlation_id=message.correlation_id)
self.socket.send_multipart(
[self.connection_id, reply.SerializeToString()], 0)
return request
#Send Event Subscription
#Connect to validator
ctx = zmq.Context()
socket = ctx.socket(zmq.DEALER)
socket.connect(url)
#Construct the request
request = ClientEventsSubscribeRequest(
subscriptions=[subscription]).SerializeToString()
#Construct the message wrapper
correlation_id = "123"
msg = Message(correlation_id=correlation_id,
message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST,
content=request)
#Send the request
socket.send_multipart([msg.SerializeToString()])
#####################################################################
#Receive the response
resp = socket.recv_multipart()[-1]
#Parse the msg wrapper
msg = Message()
msg.ParseFromString(resp)
#Validate the response type
def subscribe_event():
"""
A custom management command to store block information of each block data by event subscription.
this command is calling from docker-compose file
"""
logger.info("event function start..")
# Setup a connection to the validator
ctx = zmq.Context()
socket = ctx.socket(zmq.DEALER)
socket.connect(validator_url)
# -------------------------------#
# Submit the Event Subscription #
# -------------------------------#
# subscribe both both "block commit" and "state-delta" event
block_sub = EventSubscription(event_type='sawtooth/block-commit')
delta_sub = EventSubscription(
event_type='sawtooth/state-delta',
filters=[
EventFilter(key='address',
match_string='^{}.*'.format(NAMESPACE),
filter_type=EventFilter.REGEX_ANY)
])
# Construct the request
request = ClientEventsSubscribeRequest(
subscriptions=[block_sub, delta_sub]).SerializeToString()
# Construct the message wrapper
correlation_id = "123" # This must be unique for all in-process requests
msg = Message(correlation_id=correlation_id,
message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST,
content=request)
# Send the request
socket.send_multipart([msg.SerializeToString()])
#
# -------------------------------#
# Receive the response #
# -------------------------------#
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE:
logger.error("Unexpected message type")
return
# Parse the response
response = ClientEventsSubscribeResponse()
response.ParseFromString(msg.content)
# Validate the response status
if response.status != ClientEventsSubscribeResponse.OK:
return
while True:
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS:
return
# Parse the response
events = EventList()
events.ParseFromString(msg.content)
block_info = {"address": []}
event_data = MessageToDict(events)
events_list = event_data["events"]
for event in events_list:
attributes = event["attributes"]
for attr in attributes:
key = attr["key"]
value = attr["value"]
if key == 'address':
block_info["address"].append(value)
else:
block_info[key] = value
address_list = block_info["address"]
for address in address_list:
BlockInfo.objects.create(
block_num=block_info["block_num"],
block_id=block_info["block_id"],
state_root_hash=block_info["state_root_hash"],
previous_block_id=block_info["previous_block_id"],
address=address,
)
logger.info("blockinfo subscription created..")
def subscribe(self, event_name: str, is_write_to_file=False, file_name='certificate.pem'):
subscription = EventSubscription(event_type="ca_1/{}".format(event_name))
# Setup a connection to the validator
ctx = zmq.Context()
socket = ctx.socket(zmq.DEALER)
socket.connect('tcp://127.0.0.1:4004')
# Construct the request
request = ClientEventsSubscribeRequest(
subscriptions=[subscription]).SerializeToString()
# Construct the message wrapper
correlation_id = "123" # This must be unique for all in-process requests
msg = Message(
correlation_id=correlation_id,
message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST,
content=request)
# Send the request
socket.send_multipart([msg.SerializeToString()])
# Receive the response
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE:
print("Unexpected message type")
return
# Parse the response
response = ClientEventsSubscribeResponse()
response.ParseFromString(msg.content)
# Validate the response status
if response.status != ClientEventsSubscribeResponse.OK:
print("Subscription failed: {}".format(response.response_message))
return
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS:
print("Unexpected message type")
return
# Parse the response
events = EventList()
events.ParseFromString(msg.content)
for event in events.events:
if event.data is not None:
if is_write_to_file:
write_to_file(file_name, event.data)
else:
print(event.data)
# Construct the request
request = ClientEventsUnsubscribeRequest().SerializeToString()
# Construct the message wrapper
correlation_id = "124" # This must be unique for all in-process requests
msg = Message(
correlation_id=correlation_id,
message_type=Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST,
content=request)
# Send the request
socket.send_multipart([msg.SerializeToString()])
# Receive the response
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE:
print("Unexpected message type")
# Parse the response
response = ClientEventsUnsubscribeResponse()
response.ParseFromString(msg.content)
# Validate the response status
if response.status != ClientEventsUnsubscribeResponse.OK:
print("Unsubscription failed: {}".format(response.response_message))
# Close the connection to the validator
socket.close()
