def zmq_socket(ws):
"""
Note: Currently unused
This transfers data from ZMQ to a web socket.
The idea was to do some pre-processing in Python
before handing it to the WebSocket and consume it
in the web interface.
However from the documentation it's not very clear
how the data provided in `EventList` is formatted.
"""
LOGGER.debug("Entering ZMQ loop")
while True:
resp = socket.recv_multipart()[-1]
msg = Message()
msg.ParseFromString(resp)
if msg.message_type == Message.CLIENT_EVENTS:
events = EventList()
events.ParseFromString(msg.content)
LOGGER.debug("Received events")
LOGGER.debug(events)
# Eventually what we want to do here is ws.send(events)
# But before we have to unpack and pre-process it.
elif msg.message_type == Message.PING_REQUEST:
LOGGER.debug("Received ping request")
ws.send("ping request")
else:
LOGGER.warn("Unexpected message type '{}'".format(
msg.message_type
))
def run(self) -> None:
if not self._is_active:
LOGGER.error('Subscriber is inactive, Quitting...')
return
LOGGER.info('Starting the logging on {}'.format(self._event_log_file))
with open(self._event_log_file, 'w+', newline='') as log_file:
writer = csv.writer(log_file)
writer.writerow(['time', 'events_received'])
while self._is_active:
LOGGER.debug('Waiting for events')
message_future = self._stream.receive()
event_list = EventList()
event_list.ParseFromString(message_future.result().content)
LOGGER.debug('Received {} event(s)'.format(
len(event_list.events)))
writer.writerow([time.time(), len(event_list.events)])
self.counter += len(event_list.events)
if self.counter == self._expected_event_count:
self._is_active = False
LOGGER.debug('Existing subscription loop')
def start(self, known_ids=None):
"""Subscribes to state delta events, and then waits to receive deltas.
Sends any events received to delta handlers.
"""
if not known_ids:
known_ids = [NULL_BLOCK_ID]
self._stream.wait_for_ready()
LOGGER.debug('Subscribing to state delta events')
block_sub = EventSubscription(event_type='sawtooth/block-commit')
delta_sub = EventSubscription(
event_type='sawtooth/state-delta',
filters=[
EventFilter(key='address',
match_string='^{}.*'.format(NAMESPACE),
filter_type=EventFilter.REGEX_ANY)
])
request = ClientEventsSubscribeRequest(
last_known_block_ids=known_ids,
subscriptions=[block_sub, delta_sub])
response_future = self._stream.send(
Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST,
request.SerializeToString())
response = ClientEventsSubscribeResponse()
response.ParseFromString(response_future.result().content)
# Forked all the way back to genesis, restart with no known_ids
if (response.status == ClientEventsSubscribeResponse.UNKNOWN_BLOCK
and known_ids):
return self.start()
if response.status != ClientEventsSubscribeResponse.OK:
raise RuntimeError('Subscription failed with status: {}'.format(
ClientEventsSubscribeResponse.Status.Name(response.status)))
self._is_active = True
LOGGER.debug('Successfully subscribed to state delta events')
while self._is_active:
message_future = self._stream.receive()
event_list = EventList()
event_list.ParseFromString(message_future.result().content)
for handler in self._event_handlers:
handler(event_list.events)
def _process(self, msg: Message) -> None:
event_list = EventList()
event_list.ParseFromString(msg.content)
events = event_list.events
raw_block = next(
(e for e in events if e.event_type == "sawtooth/block-commit"), [])
block = {b.key: b.value for b in raw_block.attributes}
LOG.debug(f"Block: {block}")
event = next(
(e for e in events if e.event_type == "sawtooth/state-delta"),
None)
if event:
state_changes = StateChangeList()
state_changes.ParseFromString(event.data)
changes = list(state_changes.state_changes)
LOG.debug(f"Found {len(changes)} state changes")
self._handle_changes(block, changes)
def _create_event_payload(event_type, attributes):
return EventList(events=[
Event(event_type=Events.REMME_BATCH_DELTA.value,
attributes=[
Event.Attribute(key=attr_k, value=attr_v)
for attr_k, attr_v in attributes.items()
])
])
def listen_to_event(self):
self._stream.wait_for_ready()
# Step 1: Construct a Subscription
block_sub = EventSubscription(event_type='sawtooth/block-commit')
# Step 2: Submit the Event Subscription
request = ClientEventsSubscribeRequest(subscriptions=[block_sub])
response_future = self._stream.send(
Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST,
request.SerializeToString())
response = ClientEventsSubscribeResponse()
response.ParseFromString(response_future.result().content)
# Listen for events in an infinite loop
LOGGER.warning("Listening to events.")
while True:
msg = self._stream.receive()
event_list = EventList()
event_list.ParseFromString(msg.result().content)
for handler in self._event_handlers:
handler(event_list.events)
async def _process_msg(request, msg):
LOGGER.debug(f'Message type: {msg.message_type}')
if msg.message_type != Message.CLIENT_EVENTS:
LOGGER.debug(f'Skip unexpected msg type {msg.message_type}')
return
evt_resp = EventList()
evt_resp.ParseFromString(msg.content)
ws = request.ws
subsevt = request.rpc._subsevt.get(ws, {})
for proto_data in evt_resp.events:
evt = message_to_dict(proto_data)
LOGGER.debug(f'Dicted response evt: {evt}')
event_type = evt['event_type']
evt_names = SAWTOOTH_TO_REMME_EVENT.get(event_type)
if not evt_names:
LOGGER.debug(f'Evt names for type "{event_type}" '
'not configured')
continue
for evt_name in evt_names:
evt_tr = EVENT_HANDLERS[evt_name]
# Check if evt_name is subscribed
if evt_name not in subsevt:
LOGGER.debug('No active ws connection '
f'for evt "{evt_name}"')
continue
# Check if ws has stored hashes
if ws not in request.rpc._evthashes:
LOGGER.warning(f'Connection {ws} not found')
del subsevt[evt_name]
continue
# get response of updated state
LOGGER.debug(f'Got evt data: {evt}')
updated_state = evt_tr.parse_evt(evt)
if not updated_state:
LOGGER.debug('Skiping evt with no state update')
continue
validated_data = subsevt[evt_name]['validated_data']
LOGGER.debug(f'Loaded validated data: {validated_data}')
msg_id = subsevt[evt_name]['msg_id']
response = evt_tr.prepare_response(updated_state, validated_data)
if asyncio.iscoroutine(response):
response = await response
if not response:
LOGGER.debug('Skiping evt with empty response')
continue
LOGGER.debug(f'Got response: {response}')
evthash = evt_tr.prepare_evt_hash(response)
LOGGER.debug(f'Evt hash calculated: {evthash}')
# Check if we already have sent update
if evthash in request.rpc._evthashes[ws]:
LOGGER.debug(f'Connection {ws} already '
'received this notification')
continue
result = encode_result(msg_id, {
'event_type': evt_name,
'attributes': response
})
await request.rpc._ws_send_str(request, result)
request.rpc._evthashes[ws].add(evthash)
def subscribe(self, event_name: str, is_write_to_file=False, file_name='certificate.pem'):
subscription = EventSubscription(event_type="ca_1/{}".format(event_name))
# Setup a connection to the validator
ctx = zmq.Context()
socket = ctx.socket(zmq.DEALER)
socket.connect('tcp://127.0.0.1:4004')
# Construct the request
request = ClientEventsSubscribeRequest(
subscriptions=[subscription]).SerializeToString()
# Construct the message wrapper
correlation_id = "123" # This must be unique for all in-process requests
msg = Message(
correlation_id=correlation_id,
message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST,
content=request)
# Send the request
socket.send_multipart([msg.SerializeToString()])
# Receive the response
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE:
print("Unexpected message type")
return
# Parse the response
response = ClientEventsSubscribeResponse()
response.ParseFromString(msg.content)
# Validate the response status
if response.status != ClientEventsSubscribeResponse.OK:
print("Subscription failed: {}".format(response.response_message))
return
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS:
print("Unexpected message type")
return
# Parse the response
events = EventList()
events.ParseFromString(msg.content)
for event in events.events:
if event.data is not None:
if is_write_to_file:
write_to_file(file_name, event.data)
else:
print(event.data)
# Construct the request
request = ClientEventsUnsubscribeRequest().SerializeToString()
# Construct the message wrapper
correlation_id = "124" # This must be unique for all in-process requests
msg = Message(
correlation_id=correlation_id,
message_type=Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST,
content=request)
# Send the request
socket.send_multipart([msg.SerializeToString()])
# Receive the response
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE:
print("Unexpected message type")
# Parse the response
response = ClientEventsUnsubscribeResponse()
response.ParseFromString(msg.content)
# Validate the response status
if response.status != ClientEventsUnsubscribeResponse.OK:
print("Unsubscription failed: {}".format(response.response_message))
# Close the connection to the validator
socket.close()
def subscribe_event():
"""
A custom management command to store block information of each block data by event subscription.
this command is calling from docker-compose file
"""
logger.info("event function start..")
# Setup a connection to the validator
ctx = zmq.Context()
socket = ctx.socket(zmq.DEALER)
socket.connect(validator_url)
# -------------------------------#
# Submit the Event Subscription #
# -------------------------------#
# subscribe both both "block commit" and "state-delta" event
block_sub = EventSubscription(event_type='sawtooth/block-commit')
delta_sub = EventSubscription(
event_type='sawtooth/state-delta',
filters=[
EventFilter(key='address',
match_string='^{}.*'.format(NAMESPACE),
filter_type=EventFilter.REGEX_ANY)
])
# Construct the request
request = ClientEventsSubscribeRequest(
subscriptions=[block_sub, delta_sub]).SerializeToString()
# Construct the message wrapper
correlation_id = "123" # This must be unique for all in-process requests
msg = Message(correlation_id=correlation_id,
message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST,
content=request)
# Send the request
socket.send_multipart([msg.SerializeToString()])
#
# -------------------------------#
# Receive the response #
# -------------------------------#
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE:
logger.error("Unexpected message type")
return
# Parse the response
response = ClientEventsSubscribeResponse()
response.ParseFromString(msg.content)
# Validate the response status
if response.status != ClientEventsSubscribeResponse.OK:
return
while True:
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS:
return
# Parse the response
events = EventList()
events.ParseFromString(msg.content)
block_info = {"address": []}
event_data = MessageToDict(events)
events_list = event_data["events"]
for event in events_list:
attributes = event["attributes"]
for attr in attributes:
key = attr["key"]
value = attr["value"]
if key == 'address':
block_info["address"].append(value)
else:
block_info[key] = value
address_list = block_info["address"]
for address in address_list:
BlockInfo.objects.create(
block_num=block_info["block_num"],
block_id=block_info["block_id"],
state_root_hash=block_info["state_root_hash"],
previous_block_id=block_info["previous_block_id"],
address=address,
)
logger.info("blockinfo subscription created..")
async def check_event(self):
LOGGER.info(f"Checking for new events...")
if self.last_block_num is None:
LOGGER.debug(f"Last block number not ready!")
return
try:
msg = self._event_queue[Message.CLIENT_EVENTS].get_nowait()
except asyncio.QueueEmpty:
LOGGER.debug("No message prepared yet")
return
LOGGER.info(f"message type {msg.message_type}")
# Validate the response type
if msg.message_type != Message.CLIENT_EVENTS:
LOGGER.error("Unexpected message type")
return
# Parse the response
event_list = EventList()
event_list.ParseFromString(msg.content)
web_socks_to_notify = {}
LOGGER.debug(
f"Received the following events list: {event_list.events}")
block_num = None
block_id = None
is_event_catch_up = False
for event in event_list.events:
# assumed to be the first one in the list
if event.event_type == SAWTOOTH_BLOCK_COMMIT_EVENT_TYPE:
block_num = int(
get_value_from_key(event.attributes, "block_num"))
block_id = get_value_from_key(event.attributes, "block_id")
is_event_catch_up = block_num <= self.last_block_num
self.last_block_num = max(self.last_block_num, block_num)
else:
event_response = {}
event_response['type'] = event.event_type
event_response['transaction_id'], event_response[
'data'] = process_event(event.event_type, event.attributes)
event = self._events[event_response['type']]
for web_sock in event:
if web_sock.closed:
await self._handle_unsubscribe(web_sock)
continue
if is_event_catch_up and not event[web_sock]['is_catch_up']:
continue
if web_sock not in web_socks_to_notify:
web_socks_to_notify[web_sock] = []
web_socks_to_notify[web_sock] += [event_response]
if self.last_block_num == block_num:
self._events[event_response['type']][web_sock][
'is_catch_up'] = False
for web_sock, events in web_socks_to_notify.items():
await self._ws_send_message(
web_sock, {
Entity.EVENTS.value: events,
"block_num": block_num,
"block_id": block_id,
"last_known_block_num": self.last_block_num
})