def listen_to_events(delta_filters=None): '''Listen to cookiejar state-delta events.''' # Subscribe to events block_commit_subscription = events_pb2.EventSubscription( event_type="sawtooth/block-commit") state_delta_subscription = events_pb2.EventSubscription( event_type="sawtooth/state-delta", filters=delta_filters) fine_subscription = events_pb2.EventSubscription( event_type="AC is in good condition") problem_subscription = events_pb2.EventSubscription( event_type="AC is malfunctioning") fixed_subscription = events_pb2.EventSubscription( event_type="Maintenance fixed the AC") notfixed_subscription = events_pb2.EventSubscription( event_type="Maintenance hasn't fixed the AC yet") request = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=[fine_subscription,problem_subscription,fixed_subscription,notfixed_subscription]) # Send the subscription request stream = Stream(DEFAULT_VALIDATOR_URL) msg = stream.send(message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE # Parse the subscription response response = client_event_pb2.ClientEventsSubscribeResponse() response.ParseFromString(msg.content) assert response.status == \ client_event_pb2.ClientEventsSubscribeResponse.OK # Listen for events in an infinite loop print("Listening to events.") msg = stream.receive().result() assert msg.message_type == Message.CLIENT_EVENTS # Parse the response event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) print("Received the following events: ----------") notification=[] for event in event_list.events: notification.append((event.event_type,event.attributes)) #server_socket(notification) # Unsubscribe from events request = client_event_pb2.ClientEventsUnsubscribeRequest() msg = stream.send(Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE # Parse the unsubscribe response response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(msg.content) assert response.status == \ client_event_pb2.ClientEventsUnsubscribeResponse.OK return notification
def listen_to_events(delta_filters=None): '''Listen to cookiejar state-delta events.''' # Subscribe to events block_commit_subscription = events_pb2.EventSubscription( event_type="sawtooth/block-commit") state_delta_subscription = events_pb2.EventSubscription( event_type="sawtooth/state-delta", filters=delta_filters) bake_subscription = events_pb2.EventSubscription( event_type="cookiejar/bake") eat_subscription = events_pb2.EventSubscription(event_type="cookiejar/eat") request = client_event_pb2.ClientEventsSubscribeRequest(subscriptions=[ block_commit_subscription, state_delta_subscription, bake_subscription, eat_subscription ]) # Send the subscription request stream = Stream(DEFAULT_VALIDATOR_URL) msg = stream.send(message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE # Parse the subscription response response = client_event_pb2.ClientEventsSubscribeResponse() response.ParseFromString(msg.content) assert response.status == \ client_event_pb2.ClientEventsSubscribeResponse.OK # Listen for events in an infinite loop print("Listening to events.") while True: msg = stream.receive().result() assert msg.message_type == Message.CLIENT_EVENTS # Parse the response event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) print("Received the following events: ----------") for event in event_list.events: print(event) # Unsubscribe from events request = client_event_pb2.ClientEventsUnsubscribeRequest() msg = stream.send(Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE # Parse the unsubscribe response response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(msg.content) assert response.status == \ client_event_pb2.ClientEventsUnsubscribeResponse.OK
class Subscriber(object): def __init__(self, validator_url): LOGGER.info('Connecting to validator: %s', validator_url) self._stream = Stream(validator_url) self._event_handlers = [] def add_handler(self, handler): self._event_handlers.append(handler) def listen_to_event(self): self._stream.wait_for_ready() # Step 1: Construct a Subscription block_sub = EventSubscription(event_type='sawtooth/block-commit') # Step 2: Submit the Event Subscription request = ClientEventsSubscribeRequest(subscriptions=[block_sub]) response_future = self._stream.send( Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()) response = ClientEventsSubscribeResponse() response.ParseFromString(response_future.result().content) # Listen for events in an infinite loop LOGGER.warning("Listening to events.") while True: msg = self._stream.receive() event_list = EventList() event_list.ParseFromString(msg.result().content) for handler in self._event_handlers: handler(event_list.events)
def do_load(args): with open(args.filename, mode='rb') as fd: batches = batch_pb2.BatchList() batches.ParseFromString(fd.read()) stream = Stream(args.url) futures = [] start = time.time() for batch_list in _split_batch_list(batches): future = stream.send( message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST, content=batch_list.SerializeToString()) futures.append(future) for future in futures: result = future.result() try: assert result.message_type == Message.CLIENT_BATCH_SUBMIT_RESPONSE except ValidatorConnectionError as vce: LOGGER.warning("the future resolved to %s", vce) stop = time.time() print("batches: {} batch/sec: {}".format( str(len(batches.batches)), len(batches.batches) / (stop - start))) stream.close()
class eventHandler: def __init__(self, validatorUrl): self.stream = Stream(validatorUrl) def generateFilters(self): pass def generateSubscritionRequest(self, eventTypes, deltaFilters): susbscriptions = [] if (len(eventTypes) == len(deltaFilters)): print("same length") for i in range(0, len(events)): subscription = events_pb2.EventSubscription( event_type=event[i], filters=deltaFilters[i]) susbscriptions.append(subscription) print(susbscriptions) subscritionRequest = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=susbscriptions) return subscritionRequest else: print("Error : EventType and delta filter length mismatch ! ", flush=True) def listenToEvents(self, eventTypes, subsciptionRequest): print("here", eventTypes) message = self.stream.send( message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=subsciptionRequest.SerializeToString()).result() assert message.message_tyype == Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE, "Undefined Message Responce Type" response = client_event_pb2.ClientEventsSubscribeResponse() response.ParseFromString(message.content) assert response.status == client_event_pb2.ClientEventsSubscribeResponse.OK, "Status : Message Responce Not Okay " while True: streamMsg = self.stream.receive().result() assert streamMsg.message_type == Message.CLIENT_EVENTS, "Stream Message Type Undefined" # Parse the response eventList = events_pb2.EventList() eventList.ParseFromString(streamMsg.content) for event in eventList: if (event.event_type in eventTypes): print("Event Of Type " + eventType + " Received", flush=True) print("Event : " + event, flush=True) def unsubscribeEvent(self): request = client_event_pb2.ClientEventsUnsubscribeRequest() msg = stream.send(Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(msg.content) assert response.status == client_event_pb2.ClientEventsUnsubscribeResponse.OK print("Evens Unsubscribed ! ", flush=True)
def unsubscribe_from_events(): # Unsubscribe from events stream = Stream(DEFAULT_VALIDATOR_URL) request = client_event_pb2.ClientEventsUnsubscribeRequest() msg = stream.send(Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE # Parse the unsubscribe response response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(msg.content) assert response.status == \ client_event_pb2.ClientEventsUnsubscribeResponse.OK
def subscribe_to_events(delta_filters=None): '''Listen to attestation state-delta events.''' # Subscribe to events trust_path_subscription = events_pb2.EventSubscription( event_type="attestation/trustpath", filters=delta_filters) trust_entry_subscription = events_pb2.EventSubscription( event_type="attestation/entrypoint", filters=delta_filters) request = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=[trust_path_subscription, trust_entry_subscription]) # Send the subscription request stream = Stream(DEFAULT_VALIDATOR_URL) msg = stream.send(message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE # Parse the subscription response response = client_event_pb2.ClientEventsSubscribeResponse() response.ParseFromString(msg.content) assert response.status == \ client_event_pb2.ClientEventsSubscribeResponse.OK
class Subscriber(object): """Creates an object that can subscribe to state delta events using the Sawtooth SDK's Stream class. Handler functions can be added prior to subscribing, and each will be called on each delta event received. """ def __init__(self, validator_url): LOGGER.info("Connecting to validator: %s", validator_url) self._stream = Stream(validator_url) self._delta_handlers = [] self._is_active = False def add_handler(self, event_handler): """Adds a handler which will be passed state delta events when they occur. Note that this event is mutable. """ self._delta_handlers.append(event_handler) def clear_handlers(self): """Clears any delta handlers. """ self._delta_handlers = [] def start(self, known_ids=None): """Subscribes to state delta events, and then waits to receive deltas. Sends any events received to delta handlers. """ self._stream.wait_for_ready() LOGGER.debug("Subscribing to client state events") request = client_event_pb2.ClientEventsSubscribeRequest( last_known_block_ids=known_ids, subscriptions=[ events_pb2.EventSubscription( event_type="sawtooth/block-commit"), events_pb2.EventSubscription( event_type="sawtooth/state-delta", filters=[ events_pb2.EventFilter( key="address", match_string="^" + addresser.NAMESPACE + ".*", filter_type=events_pb2.EventFilter.REGEX_ANY, ) ], ), ], ) response_future = self._stream.send( Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()) response = client_event_pb2.ClientEventsSubscribeResponse() response.ParseFromString(response_future.result().content) # Forked all the way back to genesis, restart with no known_ids if (known_ids and response.status == client_event_pb2.ClientEventsSubscribeResponse.UNKNOWN_BLOCK): return self.start() if response.status != client_event_pb2.ClientEventsSubscribeResponse.OK: raise RuntimeError("Subscription failed with status: {}".format( client_event_pb2.ClientEventsSubscribeResponse.Status.Name( response.status))) self._is_active = True LOGGER.debug("Successfully subscribed to state delta events") while self._is_active: message_future = self._stream.receive() msg = message_future.result() if msg.message_type == Message.CLIENT_EVENTS: event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = list(event_list.events) event = StateDeltaEvent(events) delta_count = len(event.state_changes) if delta_count > 0: LOGGER.debug("Received %d deltas for block: %s", delta_count, event.block_id) for handler in self._delta_handlers: handler(event) def stop(self): """Stops the Subscriber, unsubscribing from state delta events and closing the the stream's connection. """ self._is_active = False LOGGER.debug("Unsubscribing from client events") request = client_event_pb2.ClientEventsUnsubscribeResponse() response_future = self._stream.send( Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()) response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(response_future.result().content) if response.status != client_event_pb2.ClientEventsUnsubscribeResponse.OK: LOGGER.warning( "Failed to unsubscribe with status: %s", client_event_pb2.ClientEventsUnsubscribeResponse.Status.Name( response.status), ) self._stream.close()
class SawtoothSubscriber: def __init__(self, url, benchmark, expected_event_count): self._event = '{}-{}'.format(benchmark, EVENT_SUFFIX) self._url = url self.counter = 0 self._is_active = False self._expected_event_count = expected_event_count self._stream = Stream('tcp://{}:4004'.format(self._url)) date = datetime.date.today() logging_file_name = '{}-{}-benchmark-events.csv'.format( str(date), benchmark) self._event_log_file = path.join(DEFAULT_MONITORING_FOLDER, logging_file_name) def subscribe_to_event(self): LOGGER.info("Subscribing to event {}".format(self._event)) done_sub = EventSubscription(event_type=self._event) request = ClientEventsSubscribeRequest(subscriptions=[done_sub]) response_future = self._stream.send( Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()) sub_response = ClientEventsSubscribeResponse() sub_response.ParseFromString(response_future.result().content) if sub_response.status == ClientEventsSubscribeResponse.OK: self._is_active = True LOGGER.info('Subscription with response {}'.format( sub_response.status)) else: LOGGER.error('Could not Subscribe to event') raise SubscriberError() def stop_subscription(self): self._is_active = False def run(self) -> None: if not self._is_active: LOGGER.error('Subscriber is inactive, Quitting...') return LOGGER.info('Starting the logging on {}'.format(self._event_log_file)) with open(self._event_log_file, 'w+', newline='') as log_file: writer = csv.writer(log_file) writer.writerow(['time', 'events_received']) while self._is_active: LOGGER.debug('Waiting for events') message_future = self._stream.receive() event_list = EventList() event_list.ParseFromString(message_future.result().content) LOGGER.debug('Received {} event(s)'.format( len(event_list.events))) writer.writerow([time.time(), len(event_list.events)]) self.counter += len(event_list.events) if self.counter == self._expected_event_count: self._is_active = False LOGGER.debug('Existing subscription loop')
class TestEventsAndReceipts(unittest.TestCase): def test_subscribe_and_unsubscribe(self): """Tests that a client can subscribe and unsubscribe from events.""" response = self._subscribe() self.assert_subscribe_response(response) response = self._unsubscribe() self.assert_unsubscribe_response(response) def test_block_commit_event_received(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() for i in range(1, 5): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self.assertEqual(msg.message_type, validator_pb2.Message.CLIENT_EVENTS) event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 1) self.assert_block_commit_event(events[0], i) self._unsubscribe() def test_get_events(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self._unsubscribe() event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events block_commit_event = events[0] block_id = list( filter(lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list( filter(lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value response = self._get_events( block_id, [events_pb2.EventSubscription(event_type="sawtooth/block-commit")]) events = self.assert_events_get_response(response) self.assert_block_commit_event(events[0], block_num) def test_catchup(self): """Tests that a subscriber correctly receives catchup events.""" self._subscribe() blocks = [] for i in range(4): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events block_commit_event = events[0] block_id = list( filter(lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list( filter(lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value blocks.append((block_num, block_id)) self._unsubscribe() self.assert_subscribe_response( self._subscribe(last_known_block_ids=[blocks[0][1]])) LOGGER.warning("Waiting for catchup events") for i in range(3): msg = self.stream.receive().result() LOGGER.warning("Got catchup events: ") event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 1) block_commit_event = events[0] block_id = list( filter(lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list( filter(lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value self.assertEqual((block_num, block_id), blocks[i + 1]) self._unsubscribe() def test_receipt_stored(self): """Tests that receipts are stored successfully when a block is committed.""" self._subscribe() n = self.batch_submitter.submit_next_batch() response = self._get_receipt(n) receipts = self.assert_receipt_get_response(response) state_change = receipts[0].state_changes[0] self.assertEqual(state_change.type, transaction_receipt_pb2.StateChange.SET) self.assertEqual(state_change.value, cbor.dumps({str(n): 0})) self.assertEqual(state_change.address, make_intkey_address(str(n))) self._unsubscribe() @classmethod def setUpClass(cls): cls.batch_submitter = BatchSubmitter(WAIT) def setUp(self): self.url = "tcp://validator:4004" self.stream = Stream(self.url) def tearDown(self): if self.stream is not None: self.stream.close() def _get_receipt(self, n): txn_id = \ self.batch_submitter.batches[n].transactions[0].header_signature request = client_receipt_pb2.ClientReceiptGetRequest( transaction_ids=[txn_id]) response = self.stream.send( validator_pb2.Message.CLIENT_RECEIPT_GET_REQUEST, request.SerializeToString()).result() return response def _get_events(self, block_id, subscriptions): request = client_event_pb2.ClientEventsGetRequest( block_ids=[block_id], subscriptions=subscriptions) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_GET_REQUEST, request.SerializeToString()).result() return response def _subscribe(self, subscriptions=None, last_known_block_ids=None): if subscriptions is None: subscriptions = [ events_pb2.EventSubscription( event_type="sawtooth/block-commit"), ] if last_known_block_ids is None: last_known_block_ids = [] request = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=subscriptions, last_known_block_ids=last_known_block_ids) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def _unsubscribe(self): request = client_event_pb2.ClientEventsUnsubscribeRequest() response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def assert_block_commit_event(self, event, block_num): self.assertEqual(event.event_type, "sawtooth/block-commit") self.assertTrue( all([ any(attribute.key == "block_id" for attribute in event.attributes), any(attribute.key == "block_num" for attribute in event.attributes), any(attribute.key == "previous_block_id" for attribute in event.attributes), any(attribute.key == "state_root_hash" for attribute in event.attributes), ])) for attribute in event.attributes: if attribute.key == "block_num": self.assertEqual(attribute.value, str(block_num)) def assert_receipt_get_response(self, msg): self.assertEqual(msg.message_type, validator_pb2.Message.CLIENT_RECEIPT_GET_RESPONSE) receipt_response = client_receipt_pb2.ClientReceiptGetResponse() receipt_response.ParseFromString(msg.content) self.assertEqual(receipt_response.status, client_receipt_pb2.ClientReceiptGetResponse.OK) return receipt_response.receipts def assert_events_get_response(self, msg): self.assertEqual(msg.message_type, validator_pb2.Message.CLIENT_EVENTS_GET_RESPONSE) events_response = client_event_pb2.ClientEventsGetResponse() events_response.ParseFromString(msg.content) self.assertEqual(events_response.status, client_event_pb2.ClientEventsGetResponse.OK) return events_response.events def assert_subscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE) subscription_response = client_event_pb2.ClientEventsSubscribeResponse( ) subscription_response.ParseFromString(msg.content) self.assertEqual(subscription_response.status, client_event_pb2.ClientEventsSubscribeResponse.OK) def assert_unsubscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE) subscription_response = client_event_pb2.ClientEventsUnsubscribeResponse( ) subscription_response.ParseFromString(msg.content) self.assertEqual(subscription_response.status, client_event_pb2.ClientEventsUnsubscribeResponse.OK)
class BasicClient: def __init__(self, family_handler, test_helper=None, keyfile=None): config = load_toml_with_defaults( '/config/remme-client-config.toml')['remme']['client'] self.url = config['validator_rest_api_url'] self._family_handler = family_handler self.test_helper = test_helper self._stream = Stream( f'tcp://{ config["validator_ip"] }:{ config["validator_port"] }') if keyfile is None: keyfile = PRIV_KEY_FILE try: self._signer = self.get_signer_priv_key_from_file(keyfile) except ClientException as e: LOGGER.warn('Could not set up signer from file, detailed: %s', e) self._signer = self.generate_signer(keyfile) @staticmethod def get_signer_priv_key_from_file(keyfile): try: with open(keyfile) as fd: private_key_str = fd.read().strip() except OSError as err: raise ClientException('Failed to read private key: {}'.format( str(err))) try: private_key = Secp256k1PrivateKey.from_hex(private_key_str) except ParseError as e: raise ClientException('Unable to load private key: {}'.format( str(e))) context = create_context('secp256k1') return CryptoFactory(context).new_signer(private_key) @staticmethod def generate_signer(keyfile=None): context = create_context('secp256k1') private_key = context.new_random_private_key() if keyfile: try: with open(keyfile, 'w') as fd: fd.write(private_key.as_hex()) except OSError as err: raise ClientException(f'Failed to write private key: {err}') return CryptoFactory(context).new_signer(private_key) def make_address(self, suffix): return self._family_handler.make_address(suffix) def make_address_from_data(self, data): return self._family_handler.make_address_from_data(data) def is_address(self, address): return self._family_handler.is_address(address) def get_value(self, address): result = self._send_request(f"state/{address}", conn_protocol='socket') return base64.b64decode(result['data']) def get_batch(self, batch_id): result = self._send_request(f"batch_statuses?id={batch_id}", conn_protocol='socket') return result['data'][0] def get_signer(self): return self._signer def get_public_key(self): return self.get_signer().get_public_key().as_hex() def _get_prefix(self): return self._family_handler.namespaces[-1] def _get_address(self, pub_key): if len(pub_key) > 64: raise ClientException("Wrong pub_key size: {}".format(pub_key)) prefix = self._get_prefix() return prefix + pub_key def _send_request(self, suffix, data=None, conn_protocol='text', **kwargs): if conn_protocol == 'text': return self._text_request(suffix, data, **kwargs) elif conn_protocol == 'socket': return self._socket_request(suffix, data, **kwargs) raise ClientException('Unsupported connection protocol "%s"' % conn_protocol) def _text_request(self, suffix, data=None, content_type=None): url = f"{self.url}/{suffix}" if not url.startswith("http://"): url = f"http://{url}" headers = {} if content_type is not None: headers['Content-Type'] = content_type try: if data is not None: with suppress(AttributeError): data = data.SerializeToString() result = requests.post(url, headers=headers, data=data) else: result = requests.get(url, headers=headers) if result.status_code == 404: raise KeyNotFound("404") elif not result.ok: raise ClientException("Error {}: {}".format( result.status_code, result.reason)) except requests.ConnectionError as err: raise ClientException( 'Failed to connect to REST API: {}'.format(err)) return json.loads(result.text) def _socket_request(self, suffix, data=None): if suffix == 'batches': return self.submit_batches({'batches': data.batches}) elif 'batch_statuses?id=' in suffix: _, batch_id = suffix.split('?id=') return self.get_batch_statuses({'batch_ids': [batch_id]}) elif 'state/' in suffix: _, address = suffix.split('/') _, root = self.get_root_block() return self.fetch_state({'state_root': root, 'address': address}) else: raise ClientException('Suffix "%s" not supported' % suffix) def _handle_response(self, msg_type, resp_proto, req): self._stream.wait_for_ready() future = self._stream.send(message_type=msg_type, content=req.SerializeToString()) resp = resp_proto() try: resp.ParseFromString(future.result().content) except (DecodeError, AttributeError): raise ClientException( 'Failed to parse "content" string from validator') except ValidatorConnectionError as vce: LOGGER.error('Error: %s' % vce) raise ClientException( 'Failed with ZMQ interaction: {0}'.format(vce)) data = message_to_dict(resp) # NOTE: Not all protos have this status with suppress(AttributeError): if resp.status == resp_proto.NO_RESOURCE: raise KeyNotFound("404") if resp.status != resp_proto.OK: raise ClientException("Error: %s" % data) return data def fetch_peers(self): resp = self._handle_response(Message.CLIENT_PEERS_GET_REQUEST, ClientPeersGetResponse, ClientPeersGetRequest()) return {'data': resp['peers']} def get_root_block(self): resp = self._handle_response( Message.CLIENT_BLOCK_LIST_REQUEST, ClientBlockListResponse, ClientBlockListRequest(paging=ClientPagingControls(limit=1))) block = resp['blocks'][0] header = BlockHeader() try: header_bytes = base64.b64decode(block['header']) header.ParseFromString(header_bytes) except (KeyError, TypeError, ValueError, DecodeError): header = block.get('header', None) LOGGER.error( 'The validator sent a resource with %s %s', 'a missing header' if header is None else 'an invalid header:', header or '') raise ClientException() block['header'] = message_to_dict(header) return ( block['header_signature'], block['header']['state_root_hash'], ) def submit_batches(self, data): self._handle_response( Message.CLIENT_BATCH_SUBMIT_REQUEST, ClientBatchSubmitResponse, ClientBatchSubmitRequest(batches=data['batches'])) id_string = ','.join(b.header_signature for b in data['batches']) link = f"{self.url}/batch_statuses?id={id_string}" return {'link': link} def fetch_state(self, data): resp = self._handle_response( Message.CLIENT_STATE_GET_REQUEST, ClientStateGetResponse, ClientStateGetRequest(state_root=data['state_root'], address=data['address'])) return {'data': resp['value']} def get_batch_statuses(self, data): resp = self._handle_response( Message.CLIENT_BATCH_STATUS_REQUEST, ClientBatchStatusResponse, ClientBatchStatusRequest(batch_ids=data['batch_ids'])) return {'data': resp['batch_statuses']} def make_batch_list(self, payload_pb, addresses_input, addresses_output): payload = payload_pb.SerializeToString() signer = self._signer header = TransactionHeader( signer_public_key=signer.get_public_key().as_hex(), family_name=self._family_handler.family_name, family_version=self._family_handler.family_versions[-1], inputs=addresses_input, outputs=addresses_output, dependencies=[], payload_sha512=hash512(payload), batcher_public_key=signer.get_public_key().as_hex(), nonce=time.time().hex().encode()).SerializeToString() signature = signer.sign(header) transaction = Transaction(header=header, payload=payload, header_signature=signature) return self._sign_batch_list(signer, [transaction]) def set_signer(self, new_signer): self._signer = new_signer def get_user_address(self): return AccountHandler.make_address_from_data( self._signer.get_public_key().as_hex()) def _send_transaction(self, method, data_pb, addresses_input, addresses_output): ''' Signs and sends transaction to the network using rest-api. :param str method: The method (defined in proto) for Transaction Processor to process the request. :param dict data: Dictionary that is required by TP to process the transaction. :param str addresses_input: list of addresses(keys) for which to get state. :param str addresses_output: list of addresses(keys) for which to save state. ''' addresses_input_output = [] addresses_input_output.extend(addresses_input) addresses_input_output.extend(addresses_output) addresses_input_output = list(set(addresses_input_output)) # forward transaction to test helper if self.test_helper: self.test_helper.send_transaction(method, data_pb, addresses_input_output) return payload = TransactionPayload() payload.method = method payload.data = data_pb.SerializeToString() for address in addresses_input_output: if not is_address(address): raise ClientException( 'one of addresses_input_output {} is not an address'. format(addresses_input_output)) batch_list = self.make_batch_list(payload, addresses_input, addresses_output) return get_batch_id(self._send_request('batches', batch_list, 'socket')) def _send_raw_transaction(self, transaction_pb): batch_list = self._sign_batch_list(self._signer, [transaction_pb]) return get_batch_id(self._send_request('batches', batch_list, 'socket')) def _sign_batch_list(self, signer, transactions): transaction_signatures = [t.header_signature for t in transactions] header = BatchHeader( signer_public_key=signer.get_public_key().as_hex(), transaction_ids=transaction_signatures).SerializeToString() signature = signer.sign(header) batch = Batch(header=header, transactions=transactions, header_signature=signature) return BatchList(batches=[batch])
class TransactionProcessor: """TransactionProcessor is a generic class for communicating with a validator and routing transaction processing requests to a registered handler. It uses ZMQ and channels to handle requests concurrently. """ def __init__(self, url): """ Args: url (string): The URL of the validator """ self._stream = Stream(url) self._url = url self._handlers = [] @property def zmq_id(self): return self._stream.zmq_id def add_handler(self, handler): """Adds a transaction family handler Args: handler (TransactionHandler): the handler to be added """ self._handlers.append(handler) def _matches(self, handler, header): return header.family_name == handler.family_name \ and header.family_version in handler.family_versions def _find_handler(self, header): """Find a handler for a particular (family_name, family_versions) :param header transaction_pb2.TransactionHeader: :return: handler """ try: return next( handler for handler in self._handlers if self._matches(handler, header)) except StopIteration: LOGGER.debug("Missing handler for header: %s", header) return None def _register_requests(self): """Returns all of the TpRegisterRequests for handlers :return (list): list of TpRegisterRequests """ return itertools.chain.from_iterable( # flattens the nested list [ [TpRegisterRequest( family=n, version=v, namespaces=h.namespaces) for n, v in itertools.product( [h.family_name], h.family_versions,)] for h in self._handlers]) def _unregister_request(self): """Returns a single TP_UnregisterRequest that requests that the validator stop sending transactions for previously registered handlers. :return (processor_pb2.TpUnregisterRequest): """ return TpUnregisterRequest() def _process(self, msg): if msg.message_type != Message.TP_PROCESS_REQUEST: LOGGER.debug( "Transaction Processor recieved invalid message type. " "Message type should be TP_PROCESS_REQUEST," " but is %s", Message.MessageType.Name(msg.message_type)) return request = TpProcessRequest() request.ParseFromString(msg.content) state = Context(self._stream, request.context_id) header = request.header try: if not self._stream.is_ready(): raise ValidatorConnectionError() handler = self._find_handler(header) if handler is None: return handler.apply(request, state) self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.OK ).SerializeToString()) except InvalidTransaction as it: LOGGER.warning("Invalid Transaction %s", it) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(it), extended_data=it.extended_data ).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) except InternalError as ie: LOGGER.warning("internal error: %s", ie) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INTERNAL_ERROR, message=str(ie), extended_data=ie.extended_data ).SerializeToString()) except ValidatorConnectionError as vce: # Same as the prior except block, but an internal error has # happened, but because of the disconnect the validator # probably doesn't care about the response. LOGGER.warning("during internal error response: %s", vce) except ValidatorConnectionError as vce: # Somewhere within handler.apply a future resolved with an # error status that the validator has disconnected. There is # nothing left to do but reconnect. LOGGER.warning("during handler.apply a future was resolved " "with error status: %s", vce) except AuthorizationException as ae: LOGGER.warning("AuthorizationException: %s", ae) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(ae), ).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) def _process_future(self, future, timeout=None, sigint=False): try: msg = future.result(timeout) except CancelledError: # This error is raised when Task.cancel is called on # disconnect from the validator in stream.py, for # this future. return if msg is RECONNECT_EVENT: if sigint is False: LOGGER.info("reregistering with validator") self._stream.wait_for_ready() self._register() else: LOGGER.debug( 'received message of type: %s', Message.MessageType.Name(msg.message_type)) if msg.message_type == Message.PING_REQUEST: self._stream.send_back( message_type=Message.PING_RESPONSE, correlation_id=msg.correlation_id, content=PingResponse().SerializeToString()) return self._process(msg) def _register(self): futures = [] for message in self._register_requests(): self._stream.wait_for_ready() future = self._stream.send( message_type=Message.TP_REGISTER_REQUEST, content=message.SerializeToString()) futures.append(future) for future in futures: resp = TpRegisterResponse() try: resp.ParseFromString(future.result().content) LOGGER.info("register attempt: %s", TpRegisterResponse.Status.Name(resp.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on registration: %s", vce) def _unregister(self): message = self._unregister_request() self._stream.wait_for_ready() future = self._stream.send( message_type=Message.TP_UNREGISTER_REQUEST, content=message.SerializeToString()) response = TpUnregisterResponse() try: response.ParseFromString(future.result(1).content) LOGGER.info("unregister attempt: %s", TpUnregisterResponse.Status.Name(response.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on unregistration: %s", vce) def start(self): """Connects the transaction processor to a validator and starts listening for requests and routing them to an appropriate transaction handler. """ fut = None try: self._register() while True: # During long running processing this # is where the transaction processor will # spend most of its time fut = self._stream.receive() self._process_future(fut) except KeyboardInterrupt: try: # tell the validator to not send any more messages self._unregister() while True: if fut is not None: # process futures as long as the tp has them, # if the TP_PROCESS_REQUEST doesn't come from # zeromq->asyncio in 1 second raise a # concurrent.futures.TimeOutError and be done. self._process_future(fut, 1, sigint=True) fut = self._stream.receive() except concurrent.futures.TimeoutError: # Where the tp will usually exit after # a KeyboardInterrupt. Caused by the 1 second # timeout in _process_future. pass except FutureTimeoutError: # If the validator is not able to respond to the # unregister request, exit. pass def stop(self): """Closes the connection between the TransactionProcessor and the validator. """ self._stream.close()
class RouteHandler(object): """Contains a number of aiohttp handlers for endpoints in the Rest Api. Each handler takes an aiohttp Request object, and uses the data in that request to send Protobuf message to a validator. The Protobuf response is then parsed, and finally an aiohttp Response object is sent back to the client with JSON formatted data and metadata. If something goes wrong, an aiohttp HTTP exception is raised or returned instead. Args: stream_url (str): The TCP url to communitcate with the validator timeout (int, optional): The time in seconds before the Api should cancel a request and report that the validator is unavailable. """ def __init__(self, loop, stream_url, timeout=DEFAULT_TIMEOUT): loop.set_default_executor(ThreadPoolExecutor()) self._loop = loop self._stream = Stream(stream_url) self._timeout = timeout async def submit_batches(self, request): """Accepts a binary encoded BatchList and submits it to the validator. Request: body: octet-stream BatchList of one or more Batches query: - wait: Request should not return until all batches committed Response: status: - 200: Batches submitted, but wait timed out before committed - 201: All batches submitted and committed - 202: Batches submitted and pending (not told to wait) data: Status of uncommitted batches (if any, when told to wait) link: /batches or /batch_status link for submitted batches """ # Parse request if request.headers['Content-Type'] != 'application/octet-stream': return errors.WrongBodyType() payload = await request.read() if not payload: return errors.EmptyProtobuf() try: batch_list = BatchList() batch_list.ParseFromString(payload) except DecodeError: return errors.BadProtobuf() # Query validator error_traps = [error_handlers.InvalidBatch()] validator_query = client_pb2.ClientBatchSubmitRequest( batches=batch_list.batches) self._set_wait(request, validator_query) response = await self._query_validator( Message.CLIENT_BATCH_SUBMIT_REQUEST, client_pb2.ClientBatchSubmitResponse, validator_query, error_traps) # Build response envelope data = response['batch_statuses'] or None link = '{}://{}/batch_status?id={}'.format( request.scheme, request.host, ','.join(b.header_signature for b in batch_list.batches)) if data is None: status = 202 elif any(s != 'COMMITTED' for _, s in data.items()): status = 200 else: status = 201 data = None link = link.replace('batch_status', 'batches') return self._wrap_response( data=data, metadata={'link': link}, status=status) async def list_statuses(self, request): """Fetches the committed status of batches by either a POST or GET. Request: body: A JSON array of one or more id strings (if POST) query: - id: A comma separated list of up to 15 ids (if GET) - wait: Request should not return until all batches committed Response: data: A JSON object, with batch ids as keys, and statuses as values link: The /batch_status link queried (if GET) """ error_traps = [error_handlers.StatusesNotReturned()] # Parse batch ids from POST body, or query paramaters if request.method == 'POST': if request.headers['Content-Type'] != 'application/json': return errors.BadStatusBody() ids = await request.json() if not isinstance(ids, list): return errors.BadStatusBody() if len(ids) == 0: return errors.MissingStatusId() if not isinstance(ids[0], str): return errors.BadStatusBody() else: try: ids = request.url.query['id'].split(',') except KeyError: return errors.MissingStatusId() # Query validator validator_query = client_pb2.ClientBatchStatusRequest(batch_ids=ids) self._set_wait(request, validator_query) response = await self._query_validator( Message.CLIENT_BATCH_STATUS_REQUEST, client_pb2.ClientBatchStatusResponse, validator_query, error_traps) # Send response if request.method != 'POST': metadata = self._get_metadata(request, response) else: metadata = None return self._wrap_response( data=response.get('batch_statuses'), metadata=metadata) async def list_state(self, request): """Fetches list of data leaves, optionally filtered by address prefix. Request: query: - head: The id of the block to use as the head of the chain - address: Return leaves whose addresses begin with this prefix Response: data: An array of leaf objects with address and data keys head: The head used for this query (most recent if unspecified) link: The link to this exact query, including head block paging: Paging info and nav, like total resources and a next link """ paging_controls = self._get_paging_controls(request) validator_query = client_pb2.ClientStateListRequest( head_id=request.url.query.get('head', None), address=request.url.query.get('address', None), paging=self._make_paging_message(paging_controls)) response = await self._query_validator( Message.CLIENT_STATE_LIST_REQUEST, client_pb2.ClientStateListResponse, validator_query) return self._wrap_paginated_response( request=request, response=response, controls=paging_controls, data=response.get('leaves', [])) async def fetch_state(self, request): """Fetches data from a specific address in the validator's state tree. Request: query: - head: The id of the block to use as the head of the chain - address: The 70 character address of the data to be fetched Response: data: The base64 encoded binary data stored at that address head: The head used for this query (most recent if unspecified) link: The link to this exact query, including head block """ error_traps = [ error_handlers.MissingLeaf(), error_handlers.BadAddress()] address = request.match_info.get('address', '') head = request.url.query.get('head', None) response = await self._query_validator( Message.CLIENT_STATE_GET_REQUEST, client_pb2.ClientStateGetResponse, client_pb2.ClientStateGetRequest(head_id=head, address=address), error_traps) return self._wrap_response( data=response['value'], metadata=self._get_metadata(request, response)) async def list_blocks(self, request): """Fetches list of blocks from validator, optionally filtered by id. Request: query: - head: The id of the block to use as the head of the chain - id: Comma separated list of block ids to include in results Response: data: JSON array of fully expanded Block objects head: The head used for this query (most recent if unspecified) link: The link to this exact query, including head block paging: Paging info and nav, like total resources and a next link """ paging_controls = self._get_paging_controls(request) validator_query = client_pb2.ClientBlockListRequest( head_id=request.url.query.get('head', None), block_ids=self._get_filter_ids(request), paging=self._make_paging_message(paging_controls)) response = await self._query_validator( Message.CLIENT_BLOCK_LIST_REQUEST, client_pb2.ClientBlockListResponse, validator_query) return self._wrap_paginated_response( request=request, response=response, controls=paging_controls, data=[self._expand_block(b) for b in response['blocks']]) async def fetch_block(self, request): """Fetches a specific block from the validator, specified by id. Request: path: - block_id: The 128-character id of the block to be fetched Response: data: A JSON object with the data from the fully expanded Block link: The link to this exact query """ error_traps = [error_handlers.MissingBlock()] block_id = request.match_info.get('block_id', '') response = await self._query_validator( Message.CLIENT_BLOCK_GET_REQUEST, client_pb2.ClientBlockGetResponse, client_pb2.ClientBlockGetRequest(block_id=block_id), error_traps) return self._wrap_response( data=self._expand_block(response['block']), metadata=self._get_metadata(request, response)) async def list_batches(self, request): """Fetches list of batches from validator, optionally filtered by id. Request: query: - head: The id of the block to use as the head of the chain - id: Comma separated list of batch ids to include in results Response: data: JSON array of fully expanded Batch objects head: The head used for this query (most recent if unspecified) link: The link to this exact query, including head block paging: Paging info and nav, like total resources and a next link """ paging_controls = self._get_paging_controls(request) validator_query = client_pb2.ClientBatchListRequest( head_id=request.url.query.get('head', None), batch_ids=self._get_filter_ids(request), paging=self._make_paging_message(paging_controls)) response = await self._query_validator( Message.CLIENT_BATCH_LIST_REQUEST, client_pb2.ClientBatchListResponse, validator_query) return self._wrap_paginated_response( request=request, response=response, controls=paging_controls, data=[self._expand_batch(b) for b in response['batches']]) async def fetch_batch(self, request): """Fetches a specific batch from the validator, specified by id. Request: path: - batch_id: The 128-character id of the batch to be fetched Response: data: A JSON object with the data from the fully expanded Batch link: The link to this exact query """ error_traps = [error_handlers.MissingBatch()] batch_id = request.match_info.get('batch_id', '') response = await self._query_validator( Message.CLIENT_BATCH_GET_REQUEST, client_pb2.ClientBatchGetResponse, client_pb2.ClientBatchGetRequest(batch_id=batch_id), error_traps) return self._wrap_response( data=self._expand_batch(response['batch']), metadata=self._get_metadata(request, response)) async def list_transactions(self, request): """Fetches list of txns from validator, optionally filtered by id. Request: query: - head: The id of the block to use as the head of the chain - id: Comma separated list of txn ids to include in results Response: data: JSON array of Transaction objects with expanded headers head: The head used for this query (most recent if unspecified) link: The link to this exact query, including head block paging: Paging info and nav, like total resources and a next link """ paging_controls = self._get_paging_controls(request) validator_query = client_pb2.ClientTransactionListRequest( head_id=request.url.query.get('head', None), transaction_ids=self._get_filter_ids(request), paging=self._make_paging_message(paging_controls)) response = await self._query_validator( Message.CLIENT_TRANSACTION_LIST_REQUEST, client_pb2.ClientTransactionListResponse, validator_query) data = [self._expand_transaction(t) for t in response['transactions']] return self._wrap_paginated_response( request=request, response=response, controls=paging_controls, data=data) async def fetch_transaction(self, request): """Fetches a specific transaction from the validator, specified by id. Request: path: - transaction_id: The 128-character id of the txn to be fetched Response: data: A JSON object with the data from the expanded Transaction link: The link to this exact query """ error_traps = [error_handlers.MissingTransaction()] txn_id = request.match_info.get('transaction_id', '') response = await self._query_validator( Message.CLIENT_TRANSACTION_GET_REQUEST, client_pb2.ClientTransactionGetResponse, client_pb2.ClientTransactionGetRequest(transaction_id=txn_id), error_traps) return self._wrap_response( data=self._expand_transaction(response['transaction']), metadata=self._get_metadata(request, response)) async def _query_validator(self, request_type, response_proto, content, traps=None): """Sends a request to the validator and parses the response. """ response = await self._try_validator_request(request_type, content) return self._try_response_parse(response_proto, response, traps) async def _try_validator_request(self, message_type, content): """Serializes and sends a Protobuf message to the validator. Handles timeout errors as needed. """ if isinstance(content, BaseMessage): content = content.SerializeToString() future = self._stream.send(message_type=message_type, content=content) try: response = await self._loop.run_in_executor( None, future.result, self._timeout) except FutureTimeoutError: raise errors.ValidatorUnavailable() try: return response.content # Caused by resolving a FutureError on validator disconnect except ValidatorConnectionError: raise errors.ValidatorUnavailable() @classmethod def _try_response_parse(cls, proto, response, traps=None): """Parses the Protobuf response from the validator. Uses "error traps" to send back any HTTP error triggered by a Protobuf status, both those common to many handlers, and specified individually. """ parsed = proto() parsed.ParseFromString(response) traps = traps or [] try: traps.append(error_handlers.Unknown(proto.INTERNAL_ERROR)) except AttributeError: # Not every protobuf has every status enum, so pass AttributeErrors pass try: traps.append(error_handlers.NotReady(proto.NOT_READY)) except AttributeError: pass try: traps.append(error_handlers.MissingHead(proto.NO_ROOT)) except AttributeError: pass try: traps.append(error_handlers.InvalidPaging(proto.INVALID_PAGING)) except AttributeError: pass for trap in traps: trap.check(parsed.status) return cls.message_to_dict(parsed) @staticmethod def _wrap_response(data=None, metadata=None, status=200): """Creates the JSON response envelope to be sent back to the client. """ envelope = metadata or {} if data is not None: envelope['data'] = data return web.Response( status=status, content_type='application/json', text=json.dumps( envelope, indent=2, separators=(',', ': '), sort_keys=True)) @classmethod def _wrap_paginated_response(cls, request, response, controls, data): """Builds the metadata for a pagingated response and wraps everying in a JSON encoded web.Response """ head = response['head_id'] link = cls._build_url(request, head) paging_response = response['paging'] total = paging_response['total_resources'] paging = {'total_count': total} # If there are no resources, there should be nothing else in paging if total == 0: return cls._wrap_response( data=data, metadata={'head': head, 'link': link, 'paging': paging}) count = controls.get('count', len(data)) start = paging_response['start_index'] paging['start_index'] = start # Builds paging urls specific to this response def build_pg_url(min_pos=None, max_pos=None): return cls._build_url(request, head, count, min_pos, max_pos) # Build paging urls based on ids if 'start_id' in controls or 'end_id' in controls: if paging_response['next_id']: paging['next'] = build_pg_url(paging_response['next_id']) if paging_response['previous_id']: paging['previous'] = build_pg_url( max_pos=paging_response['previous_id']) # Build paging urls based on indexes else: end_index = controls.get('end_index', None) if end_index is None and start + count < total: paging['next'] = build_pg_url(start + count) elif end_index is not None and end_index + 1 < total: paging['next'] = build_pg_url(end_index + 1) if start - count >= 0: paging['previous'] = build_pg_url(start - count) return cls._wrap_response( data=data, metadata={'head': head, 'link': link, 'paging': paging}) @classmethod def _get_metadata(cls, request, response): """Parses out the head and link properties based on the HTTP Request from the client, and the Protobuf response from the validator. """ head = response.get('head_id', None) metadata = {'link': cls._build_url(request, head)} if head is not None: metadata['head'] = head return metadata @classmethod def _build_url(cls, request, head=None, count=None, min_pos=None, max_pos=None): """Builds a response URL to send back in response envelope. """ query = request.url.query.copy() if head is not None: url = '{}://{}{}?head={}'.format( request.scheme, request.host, request.path, head) query.pop('head', None) else: return str(request.url) if min_pos is not None: url += '&{}={}'.format('min', min_pos) elif max_pos is not None: url += '&{}={}'.format('max', max_pos) else: queries = ['{}={}'.format(k, v) for k, v in query.items()] return url + '&' + '&'.join(queries) if queries else url url += '&{}={}'.format('count', count) query.pop('min', None) query.pop('max', None) query.pop('count', None) queries = ['{}={}'.format(k, v) for k, v in query.items()] return url + '&' + '&'.join(queries) if queries else url @classmethod def _expand_block(cls, block): """Deserializes a Block's header, and the header of its Batches. """ cls._parse_header(BlockHeader, block) if 'batches' in block: block['batches'] = [cls._expand_batch(b) for b in block['batches']] return block @classmethod def _expand_batch(cls, batch): """Deserializes a Batch's header, and the header of its Transactions. """ cls._parse_header(BatchHeader, batch) if 'transactions' in batch: batch['transactions'] = [ cls._expand_transaction(t) for t in batch['transactions']] return batch @classmethod def _expand_transaction(cls, transaction): """Deserializes a Transaction's header. """ return cls._parse_header(TransactionHeader, transaction) @classmethod def _parse_header(cls, header_proto, obj): """Deserializes a base64 encoded Protobuf header. """ header = header_proto() header_bytes = base64.b64decode(obj['header']) header.ParseFromString(header_bytes) obj['header'] = cls.message_to_dict(header) return obj @staticmethod def _get_paging_controls(request): """Parses min, max, and/or count queries into A paging controls dict. """ min_pos = request.url.query.get('min', None) max_pos = request.url.query.get('max', None) count = request.url.query.get('count', None) controls = {} if count == '0': raise errors.BadCount() elif count is not None: try: controls['count'] = int(count) except ValueError: raise errors.BadCount() if min_pos is not None: try: controls['start_index'] = int(min_pos) except ValueError: controls['start_id'] = min_pos elif max_pos is not None: try: controls['end_index'] = int(max_pos) except ValueError: controls['end_id'] = max_pos return controls @staticmethod def _make_paging_message(controls): """Turns a raw paging controls dict into Protobuf PagingControls. """ count = controls.get('count', None) end_index = controls.get('end_index', None) # an end_index must be changed to start_index, possibly modifying count if end_index is not None: if count is None: start_index = 0 count = end_index elif count > end_index + 1: start_index = 0 count = end_index + 1 else: start_index = end_index + 1 - count else: start_index = controls.get('start_index', None) return client_pb2.PagingControls( start_id=controls.get('start_id', None), end_id=controls.get('end_id', None), start_index=start_index, count=count) def _set_wait(self, request, validator_query): """Parses the `wait` query parameter, and sets the corresponding `wait_for_commit` and `timeout` properties in the validator query. """ wait = request.url.query.get('wait', 'false') if wait.lower() != 'false': validator_query.wait_for_commit = True try: validator_query.timeout = int(wait) except ValueError: # By default, waits for 95% of REST API's configured timeout validator_query.timeout = int(self._timeout * 0.95) @staticmethod def _get_filter_ids(request): """Parses the `id` filter paramter from the url query. """ filter_ids = request.url.query.get('id', None) return filter_ids and filter_ids.split(',') @staticmethod def message_to_dict(message): """Converts a Protobuf object to a python dict with desired settings. """ return MessageToDict( message, including_default_value_fields=True, preserving_proto_field_name=True)
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): self._stream = Stream(endpoint) startup_state = self._register() # Validators version 1.1 send startup info with the registration # response; newer versions will send an activation message with the # startup info if startup_state is None: startup_state = self._wait_until_active() self._updates = Queue() driver_thread = Thread( target=self._driver_loop) driver_thread.start() try: self._engine.start( self._updates, ZmqService( stream=self._stream, timeout=SERVICE_TIMEOUT), startup_state) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught engine exception") self.stop() driver_thread.join() def _driver_loop(self): try: future = self._stream.receive() while True: if self._exit: self._engine.stop() break try: message = future.result(1) future = self._stream.receive() except concurrent.futures.TimeoutError: continue try: result = self._process(message) # if message was a ping ignore if result[0] == Message.PING_REQUEST: continue self._updates.put(result) except exceptions.ReceiveError as err: LOGGER.warning("%s", err) continue except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught driver exception") def stop(self): self._exit = True self._engine.stop() self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ) for (name, version) in self._engine.additional_protocols(): protocol = request.additional_protocols.add() protocol.name = name protocol.version = version while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request.SerializeToString()) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if ( response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY ): continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: if ( response.HasField('chain_head') and response.HasField('local_peer_info') ): return StartupState( response.chain_head, response.peers, response.local_peer_info) return None raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _wait_until_active(self): future = self._stream.receive() while True: try: message = future.result(1) except concurrent.futures.TimeoutError: continue if ( message.message_type == Message.CONSENSUS_NOTIFY_ENGINE_ACTIVATED ): notification = \ consensus_pb2.ConsensusNotifyEngineActivated() notification.ParseFromString(message.content) startup_state = StartupState( notification.chain_head, notification.peers, notification.local_peer_info) LOGGER.info( 'Received activation message with startup state: %s', startup_state) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck() .SerializeToString()) return startup_state LOGGER.warning('Received message type %s while waiting for \ activation message', message.message_type) future = self._stream.receive() def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) header = consensus_pb2.ConsensusPeerMessageHeader() header.ParseFromString(notification.message.header) peer_message = PeerMessage( header=header, header_bytes=notification.message.header, header_signature=notification.message.header_signature, content=notification.message.content) data = peer_message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_ENGINE_DEACTIVATED: self.stop() data = None elif type_tag == Message.PING_REQUEST: data = None else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
class Subscriber(object): """Creates an object that can subscribe to state delta events using the Sawtooth SDK's Stream class. Handler functions can be added prior to subscribing, and each will be called on each delta event received. """ def __init__(self, validator_url): LOGGER.info('Connecting to validator: %s', validator_url) self._stream = Stream(validator_url) self._delta_handlers = [] self._is_active = False def add_handler(self, event_handler): """Adds a handler which will be passed state delta events when they occur. Note that this event is mutable. """ self._delta_handlers.append(event_handler) def clear_handlers(self): """Clears any delta handlers. """ self._delta_handlers = [] def start(self, known_ids=None): """Subscribes to state delta events, and then waits to receive deltas. Sends any events received to delta handlers. """ self._stream.wait_for_ready() LOGGER.debug('Subscribing to state delta events') request = StateDeltaSubscribeRequest(last_known_block_ids=known_ids, address_prefixes=[NAMESPACE]) response_future = self._stream.send( Message.STATE_DELTA_SUBSCRIBE_REQUEST, request.SerializeToString()) response = StateDeltaSubscribeResponse() response.ParseFromString(response_future.result().content) # Forked all the way back to genesis, restart with no known_ids if (known_ids and response.status == StateDeltaSubscribeResponse.UNKNOWN_BLOCK): return self.start() if response.status != StateDeltaSubscribeResponse.OK: raise RuntimeError('Subscription failed with status: {}'.format( StateDeltaSubscribeResponse.Status.Name(response.status))) self._is_active = True LOGGER.debug('Successfully subscribed to state delta events') while self._is_active: message_future = self._stream.receive() event = StateDeltaEvent() event.ParseFromString(message_future.result().content) LOGGER.debug('Received deltas for block: %s', event.block_id) for handler in self._delta_handlers: handler(event) def stop(self): """Stops the Subscriber, unsubscribing from state delta events and closing the the stream's connection. """ self._is_active = False LOGGER.debug('Unsubscribing from state delta events') request = StateDeltaUnsubscribeRequest() response_future = self._stream.send( Message.STATE_DELTA_UNSUBSCRIBE_REQUEST, request.SerializeToString()) response = StateDeltaUnsubscribeResponse() response.ParseFromString(response_future.result().content) if response.status != StateDeltaUnsubscribeResponse.OK: LOGGER.warning( 'Failed to unsubscribe with status: %s', StateDeltaUnsubscribeResponse.Status.Name(response.status)) self._stream.close()
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): LOGGER.debug('ZmqDriver: start endpoint=%s', endpoint) self._stream = Stream(endpoint) startup_state = self._register() self._updates = Queue() driver_thread = Thread(target=self._driver_loop) driver_thread.start() try: self._engine.start( self._updates, ZmqService(stream=self._stream, timeout=SERVICE_TIMEOUT, name=self._engine.name(), version=self._engine.version()), startup_state) except Exception as ex: # pylint: disable=broad-except LOGGER.exception("Uncaught engine exception(%s)", ex) self.stop() driver_thread.join() def _driver_loop(self): try: future = self._stream.receive() LOGGER.debug('ZmqDriver: _driver_loop future=%s', future) while True: if self._exit: self._engine.stop() break try: message = future.result(1) future = self._stream.receive() except concurrent.futures.TimeoutError: continue #LOGGER.debug('ZmqDriver: _driver_loop _process') result = self._process(message) self._updates.put(result) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught driver exception") def stop(self): self._exit = True self._engine.stop() self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ).SerializeToString() while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if (response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY): #LOGGER.debug('ZmqDriver:register NOT_READY: url=%s',self._stream._url) continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: LOGGER.debug('ZmqDriver:register DONE: url=%s', self._stream._url) return StartupState(response.chain_head, response.peers, response.local_peer_info, response.peering_mode) raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info, notification.status, notification.mode, notification.info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) data = notification.message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
class TransactionProcessor: """TransactionProcessor is a generic class for communicating with a validator and routing transaction processing requests to a registered handler. It uses ZMQ and channels to handle requests concurrently. """ # This is the version used by SDK to match if validator supports feature # it requested during registration. It should only be incremented when # there are changes in TpRegisterRequest. Remember to sync this # information in validator if changed. # Note: SDK_PROTOCOL_VERSION is the highest version the SDK supports class _FeatureVersion(Enum): FEATURE_UNUSED = 0 FEATURE_CUSTOM_HEADER_STYLE = 1 SDK_PROTOCOL_VERSION = 1 def __init__(self, url): """ Args: url (string): The URL of the validator """ self._stream = Stream(url) self._url = url self._handlers = [] self._highest_sdk_feature_requested = \ self._FeatureVersion.FEATURE_UNUSED self._header_style = TpRegisterRequest.HEADER_STYLE_UNSET @property def zmq_id(self): return self._stream.zmq_id def add_handler(self, handler): """Adds a transaction family handler Args: handler (TransactionHandler): the handler to be added """ self._handlers.append(handler) def set_header_style(self, style): """Sets a flag to request the validator for custom transaction header style in TpProcessRequest. Args: style (TpProcessRequestHeaderStyle): enum value to set header style """ if self._FeatureVersion.FEATURE_CUSTOM_HEADER_STYLE.value > \ self._highest_sdk_feature_requested.value: self._highest_sdk_feature_requested = \ self._FeatureVersion.FEATURE_CUSTOM_HEADER_STYLE self._header_style = style def _matches(self, handler, header): return header.family_name == handler.family_name \ and header.family_version in handler.family_versions def _find_handler(self, header): """Find a handler for a particular (family_name, family_versions) :param header transaction_pb2.TransactionHeader: :return: handler """ try: return next(handler for handler in self._handlers if self._matches(handler, header)) except StopIteration: LOGGER.debug("Missing handler for header: %s", header) return None def _register_requests(self): """Returns all of the TpRegisterRequests for handlers :return (list): list of TpRegisterRequests """ return itertools.chain.from_iterable( # flattens the nested list [[ TpRegisterRequest( family=n, version=v, namespaces=h.namespaces, protocol_version=self._highest_sdk_feature_requested.value, request_header_style=self._header_style) for n, v in itertools.product( [h.family_name], h.family_versions, ) ] for h in self._handlers]) def _unregister_request(self): """Returns a single TP_UnregisterRequest that requests that the validator stop sending transactions for previously registered handlers. :return (processor_pb2.TpUnregisterRequest): """ return TpUnregisterRequest() def _process(self, msg): if msg.message_type != Message.TP_PROCESS_REQUEST: LOGGER.debug( "Transaction Processor recieved invalid message type. " "Message type should be TP_PROCESS_REQUEST," " but is %s", Message.MessageType.Name(msg.message_type)) return request = TpProcessRequest() request.ParseFromString(msg.content) state = Context(self._stream, request.context_id) if self._header_style == TpRegisterRequest.RAW: header = TransactionHeader() header.ParseFromString(request.header_bytes) else: header = request.header try: if not self._stream.is_ready(): raise ValidatorConnectionError() handler = self._find_handler(header) if handler is None: return handler.apply(request, state) self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.OK).SerializeToString()) except InvalidTransaction as it: LOGGER.warning("Invalid Transaction %s", it) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(it), extended_data=it.extended_data).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) except InternalError as ie: LOGGER.warning("internal error: %s", ie) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INTERNAL_ERROR, message=str(ie), extended_data=ie.extended_data).SerializeToString()) except ValidatorConnectionError as vce: # Same as the prior except block, but an internal error has # happened, but because of the disconnect the validator # probably doesn't care about the response. LOGGER.warning("during internal error response: %s", vce) except ValidatorConnectionError as vce: # Somewhere within handler.apply a future resolved with an # error status that the validator has disconnected. There is # nothing left to do but reconnect. LOGGER.warning( "during handler.apply a future was resolved " "with error status: %s", vce) except AuthorizationException as ae: LOGGER.warning("AuthorizationException: %s", ae) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(ae), ).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) def _process_future(self, future, timeout=None, sigint=False): try: msg = future.result(timeout) except CancelledError: # This error is raised when Task.cancel is called on # disconnect from the validator in stream.py, for # this future. return if msg is RECONNECT_EVENT: if sigint is False: LOGGER.info("reregistering with validator") self._stream.wait_for_ready() self._register() else: LOGGER.debug('received message of type: %s', Message.MessageType.Name(msg.message_type)) if msg.message_type == Message.PING_REQUEST: self._stream.send_back( message_type=Message.PING_RESPONSE, correlation_id=msg.correlation_id, content=PingResponse().SerializeToString()) return self._process(msg) def _register(self): futures = [] for message in self._register_requests(): self._stream.wait_for_ready() future = self._stream.send( message_type=Message.TP_REGISTER_REQUEST, content=message.SerializeToString()) futures.append(future) for future in futures: resp = TpRegisterResponse() try: resp.ParseFromString(future.result().content) if resp.protocol_version != \ self._highest_sdk_feature_requested.value: LOGGER.error( "Validator version %s does not support " "requested feature by SDK version %s. " "Unregistering with the validator.", str(resp.protocol_version), str(self._highest_sdk_feature_requested.value)) raise ValidatorVersionError() LOGGER.info("register attempt: %s", TpRegisterResponse.Status.Name(resp.status)) if resp.status == TpRegisterResponse.ERROR: raise RuntimeError("Transaction processor registration " "failed") except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on registration: %s", vce) def _unregister(self): message = self._unregister_request() self._stream.wait_for_ready() future = self._stream.send(message_type=Message.TP_UNREGISTER_REQUEST, content=message.SerializeToString()) response = TpUnregisterResponse() try: response.ParseFromString(future.result(1).content) LOGGER.info("unregister attempt: %s", TpUnregisterResponse.Status.Name(response.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on unregistration: %s", vce) def start(self): """Connects the transaction processor to a validator and starts listening for requests and routing them to an appropriate transaction handler. """ fut = None try: self._register() while True: # During long running processing this # is where the transaction processor will # spend most of its time fut = self._stream.receive() self._process_future(fut) except (KeyboardInterrupt, ValidatorVersionError): try: # tell the validator to not send any more messages self._unregister() while True: if fut is not None: # process futures as long as the tp has them, # if the TP_PROCESS_REQUEST doesn't come from # zeromq->asyncio in 1 second raise a # concurrent.futures.TimeOutError and be done. self._process_future(fut, 1, sigint=True) fut = self._stream.receive() except concurrent.futures.TimeoutError: # Where the tp will usually exit after # a KeyboardInterrupt. Caused by the 1 second # timeout in _process_future. pass except FutureTimeoutError: # If the validator is not able to respond to the # unregister request, exit. pass except RuntimeError as e: LOGGER.error("Error: %s", e) self.stop() def stop(self): """Closes the connection between the TransactionProcessor and the validator. """ self._stream.close()
class TestEventBroadcaster(unittest.TestCase): def test_subscribe_and_unsubscribe(self): """Tests that a client can subscribe and unsubscribe from events.""" response = self._subscribe() self.assert_subscribe_response(response) response = self._unsubscribe() self.assert_unsubscribe_response(response) def test_block_commit_event_received(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() for i in range(1, 5): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self.assertEqual(msg.message_type, validator_pb2.Message.CLIENT_EVENTS) event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 1) self.assert_block_commit_event(events[0], i) @classmethod def setUpClass(cls): cls.batch_submitter = BatchSubmitter() def setUp(self): self.url = "tcp://validator:4004" self.stream = Stream(self.url) def tearDown(self): if self.stream is not None: self.stream.close() def _subscribe(self, subscriptions=None): if subscriptions is None: subscriptions = [ events_pb2.EventSubscription(event_type="block_commit"), ] request = events_pb2.ClientEventsSubscribeRequest( subscriptions=subscriptions) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def _unsubscribe(self): request = events_pb2.ClientEventsUnsubscribeRequest() response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def assert_block_commit_event(self, event, block_num): self.assertEqual(event.event_type, "block_commit") self.assertTrue( all([ any(attribute.key == "block_id" for attribute in event.attributes), any(attribute.key == "block_num" for attribute in event.attributes), any(attribute.key == "previous_block_id" for attribute in event.attributes), any(attribute.key == "state_root_hash" for attribute in event.attributes), ])) for attribute in event.attributes: if attribute.key == "block_num": self.assertEqual(attribute.value, str(block_num)) def assert_subscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE) subscription_response = events_pb2.ClientEventsSubscribeResponse() subscription_response.ParseFromString(msg.content) self.assertEqual(subscription_response.status, events_pb2.ClientEventsSubscribeResponse.OK) def assert_unsubscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE) subscription_response = events_pb2.ClientEventsUnsubscribeResponse() subscription_response.ParseFromString(msg.content) self.assertEqual(subscription_response.status, events_pb2.ClientEventsUnsubscribeResponse.OK)
class TestEventsAndReceipts(unittest.TestCase): def test_subscribe_and_unsubscribe(self): """Tests that a client can subscribe and unsubscribe from events.""" response = self._subscribe() self.assert_subscribe_response(response) response = self._unsubscribe() self.assert_unsubscribe_response(response) def test_subscribe_and_unsubscribe_with_catch_up(self): """Tests that a client can subscribe and unsubscribe from events.""" response = self._subscribe( last_known_block_ids=[NULL_BLOCK_IDENTIFIER]) self.assert_subscribe_response(response) # Ensure that it receives the genesis block msg = self.stream.receive().result() self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS) event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 2) self.assert_block_commit_event(events[0], 0) self.assert_state_event(events[1], '000000') response = self._unsubscribe() self.assert_unsubscribe_response(response) def test_block_commit_event_received(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() for i in range(1, 5): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS) event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 2) self.assert_block_commit_event(events[0], i) self.assert_state_event(events[1], INTKEY_ADDRESS_PREFIX) self._unsubscribe() def test_get_events(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self._unsubscribe() event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events block_commit_event = events[0] block_id = list(filter( lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list(filter( lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value response = self._get_events( block_id, [events_pb2.EventSubscription(event_type="sawtooth/block-commit")]) events = self.assert_events_get_response(response) self.assert_block_commit_event(events[0], block_num) def test_catchup(self): """Tests that a subscriber correctly receives catchup events.""" self._subscribe() blocks = [] for i in range(4): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events block_commit_event = events[0] block_id = list(filter( lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list(filter( lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value blocks.append((block_num, block_id)) self._unsubscribe() self.assert_subscribe_response( self._subscribe(last_known_block_ids=[blocks[0][1]])) LOGGER.warning("Waiting for catchup events") for i in range(3): msg = self.stream.receive().result() LOGGER.warning("Got catchup events: ") event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 2) block_commit_event = events[0] block_id = list(filter( lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list(filter( lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value self.assertEqual((block_num, block_id), blocks[i + 1]) self._unsubscribe() def test_receipt_stored(self): """Tests that receipts are stored successfully when a block is committed.""" self._subscribe() n = self.batch_submitter.submit_next_batch() response = self._get_receipt(n) receipts = self.assert_receipt_get_response(response) state_change = receipts[0].state_changes[0] self.assertEqual( state_change.type, transaction_receipt_pb2.StateChange.SET) self.assertEqual( state_change.value, cbor.dumps({str(n): 0})) self.assertEqual( state_change.address, make_intkey_address(str(n))) self._unsubscribe() @classmethod def setUpClass(cls): wait_for_rest_apis(['rest-api:8008']) cls.batch_submitter = BatchSubmitter(WAIT) def setUp(self): self.url = "tcp://validator:4004" self.stream = Stream(self.url) def tearDown(self): if self.stream is not None: self.stream.close() def _get_receipt(self, num): txn_id = \ self.batch_submitter.batches[num].transactions[0].header_signature request = client_receipt_pb2.ClientReceiptGetRequest( transaction_ids=[txn_id]) response = self.stream.send( validator_pb2.Message.CLIENT_RECEIPT_GET_REQUEST, request.SerializeToString()).result() return response def _get_events(self, block_id, subscriptions): request = client_event_pb2.ClientEventsGetRequest( block_ids=[block_id], subscriptions=subscriptions) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_GET_REQUEST, request.SerializeToString()).result() return response def _subscribe(self, subscriptions=None, last_known_block_ids=None): if subscriptions is None: subscriptions = [ events_pb2.EventSubscription( event_type="sawtooth/block-commit"), # Subscribe to the settings state events, to test genesis # catch-up. events_pb2.EventSubscription( event_type="sawtooth/state-delta", filters=[events_pb2.EventFilter( key='address', match_string='000000.*', filter_type=events_pb2.EventFilter.REGEX_ANY)]), # Subscribe to the intkey state events, to test additional # events. events_pb2.EventSubscription( event_type="sawtooth/state-delta", filters=[events_pb2.EventFilter( key='address', match_string='{}.*'.format(INTKEY_ADDRESS_PREFIX), filter_type=events_pb2.EventFilter.REGEX_ANY)]), ] if last_known_block_ids is None: last_known_block_ids = [] request = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=subscriptions, last_known_block_ids=last_known_block_ids) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def _unsubscribe(self): request = client_event_pb2.ClientEventsUnsubscribeRequest() response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def assert_block_commit_event(self, event, block_num): self.assertEqual(event.event_type, "sawtooth/block-commit") self.assertTrue( all([ any(attribute.key == "block_id" for attribute in event.attributes), any(attribute.key == "block_num" for attribute in event.attributes), any(attribute.key == "previous_block_id" for attribute in event.attributes), any(attribute.key == "state_root_hash" for attribute in event.attributes), ])) for attribute in event.attributes: if attribute.key == "block_num": self.assertEqual(attribute.value, str(block_num)) def assert_receipt_get_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_RECEIPT_GET_RESPONSE) receipt_response = client_receipt_pb2.ClientReceiptGetResponse() receipt_response.ParseFromString(msg.content) self.assertEqual( receipt_response.status, client_receipt_pb2.ClientReceiptGetResponse.OK) return receipt_response.receipts def assert_state_event(self, event, address_prefix): self.assertEqual(event.event_type, "sawtooth/state-delta") state_change_list = transaction_receipt_pb2.StateChangeList() state_change_list.ParseFromString(event.data) for change in state_change_list.state_changes: self.assertTrue(change.address.startswith(address_prefix)) def assert_events_get_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_GET_RESPONSE) events_response = client_event_pb2.ClientEventsGetResponse() events_response.ParseFromString(msg.content) self.assertEqual( events_response.status, client_event_pb2.ClientEventsGetResponse.OK) return events_response.events def assert_subscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE) response = client_event_pb2.ClientEventsSubscribeResponse() response.ParseFromString(msg.content) self.assertEqual( response.status, client_event_pb2.ClientEventsSubscribeResponse.OK) def assert_unsubscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE) response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(msg.content) self.assertEqual( response.status, client_event_pb2.ClientEventsUnsubscribeResponse.OK)
def listen_to_events(delta_filters=None): BenLogDB.set_table('beneficiary_log', '(id,beneficiary_id,beneficiary_type_id)') '''Listen to vaccination state-delta events.''' # Subscribe to events add_beneficiary_subscription = events_pb2.EventSubscription( event_type="Beneficiary/Add_Beneficiary", filters=delta_filters) block_commit_subscription = events_pb2.EventSubscription( event_type="sawtooth/block-commit", filters=delta_filters) #Create subscription request requestBen = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=[ block_commit_subscription, add_beneficiary_subscription ], last_known_block_ids=['0000000000000000']) # Send the subscription request streamBen = Stream(BEN_VALIDATOR_URL) msgBen = streamBen.send( message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=requestBen.SerializeToString()).result() assert msgBen.message_type == Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE # Parse the subscription response responseBen = client_event_pb2.ClientEventsSubscribeResponse() responseBen.ParseFromString(msgBen.content) assert responseBen.status == client_event_pb2.ClientEventsSubscribeResponse.OK # Listen for events in an infinite loop while True: msgBen = streamBen.receive().result() assert msgBen.message_type == Message.CLIENT_EVENTS # Parse the response event_list_ben = events_pb2.EventList() event_list_ben.ParseFromString(msgBen.content) # Log each Beneficiary event into the DB for event in event_list_ben.events: if event.event_type == "Beneficiary/Add_Beneficiary": print("Received the beneficiry event", flush=True) print("Beneficiary ID : ", event.attributes[0].value, flush=True) print("Beneficiary Type : ", event.attributes[1].value, flush=True) BenLogDB.insert_data( uuid.uuid4(), #uuid event.attributes[0].value, #beneficiaryId event.attributes[1].value) #beneficiaryType # Unsubscribe from events request = client_event_pb2.ClientEventsUnsubscribeRequest() msg = stream.send(Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE # Parse the unsubscribe response response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(msg.content) assert response.status == \ client_event_pb2.ClientEventsUnsubscribeResponse.OK
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): self._stream = Stream(endpoint) (chain_head, peers) = self._register() self._updates = Queue() engine_thread = Thread( target=self._engine.start, args=(self._updates, ZmqService(stream=self._stream, timeout=SERVICE_TIMEOUT, name=self._engine.name(), version=self._engine.version()), chain_head, peers)) engine_thread.start() while True: if self._exit: self._engine.stop() engine_thread.join() break try: message = self._stream.receive().result(0.1) except concurrent.futures.TimeoutError: continue result = self._process(message) self._updates.put(result) def stop(self): self._exit = True self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ).SerializeToString() while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if (response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY): continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: return (response.chain_head, response.peers) raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) data = notification.message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
class SyncProcessor: future: Optional["Future"] = None _stream: Stream def __init__(self, url=None): self._stream = Stream(url or VALIDATOR_URL) self._handlers = [] def add_handler(self, handler: Type["HandlerBase"]): if handler not in self._handlers: LOG.info(f"Add handler {handler.__name__}") self._handlers.append(handler) def start(self): LOG.info("Start SyncProcessor") future = None try: self._register() while True: future = self._stream.receive() self._process_future(future) except KeyboardInterrupt: LOG.warning("break") try: # tell the validator to not send any more messages self._unregister() while True: if future is not None: # process futures as long as the tp has them, # if the TP_PROCESS_REQUEST doesn't come from # zeromq->asyncio in 1 second raise a # concurrent.futures.TimeOutError and be done. self._process_future(future, 1, sigint=True) future = self._stream.receive() except ConcurrentTimeoutError: # Where the tp will usually exit after # a KeyboardInterrupt. Caused by the 1 second # timeout in _process_future. pass except FutureTimeoutError: # If the validator is not able to respond to the # unregister request, exit. pass def _unregister(self): LOG.warning("unregister") pass def _register(self): LOG.debug(f"Register handlers") self._stream.wait_for_ready() future = self._stream.send( message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=ClientEventsSubscribeRequest( last_known_block_ids=[NULL_BLOCK_ID], subscriptions=[ EventSubscription(event_type="sawtooth/block-commit"), EventSubscription( event_type="sawtooth/state-delta", filters=[ EventFilter( key="address", match_string=f"^{Namespaces.GA_NAMESPACE}.*", filter_type=EventFilter.REGEX_ANY, ) ], ), ], ).SerializeToString(), ) resp = TpRegisterResponse() try: resp.ParseFromString(future.result().content) LOG.info("Register response: %s", TpRegisterResponse.Status.Name(resp.status)) except ValidatorConnectionError as vce: LOG.info("During waiting for response on registration: %s", vce) except Exception as e: LOG.info("During waiting for response on registration: %s", e) def _process_future(self, future: "Future", timeout=None, sigint=False) -> None: try: msg = future.result(timeout) except CancelledError: # This error is raised when Task.cancel is called on # disconnect from the validator in stream.py, for # this future. LOG.error("Cancelled") return if msg is RECONNECT_EVENT: if sigint is False: LOG.info("Reregistering with validator") self._stream.wait_for_ready() self._register() else: LOG.debug(f"Received message of type: " f"{Message.MessageType.Name(msg.message_type)}") if msg.message_type == Message.PING_REQUEST: self._stream.send_back( message_type=Message.PING_RESPONSE, correlation_id=msg.correlation_id, content=PingResponse().SerializeToString(), ) return elif msg.message_type == Message.CLIENT_EVENTS: self._process(msg) return LOG.warning(f"SyncProcessor got wrong message type: " f"{Message.MessageType.Name(msg.message_type)}") def _process(self, msg: Message) -> None: event_list = EventList() event_list.ParseFromString(msg.content) events = event_list.events raw_block = next( (e for e in events if e.event_type == "sawtooth/block-commit"), []) block = {b.key: b.value for b in raw_block.attributes} LOG.debug(f"Block: {block}") event = next( (e for e in events if e.event_type == "sawtooth/state-delta"), None) if event: state_changes = StateChangeList() state_changes.ParseFromString(event.data) changes = list(state_changes.state_changes) LOG.debug(f"Found {len(changes)} state changes") self._handle_changes(block, changes) def _handle_changes(self, block: dict, changes: list): # r.table(Tbl.BLOCKS).insert(block).run() for change in changes: for handler in self._handlers: if change.address.startswith(handler.prefix): handler.process(change)
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): self._stream = Stream(endpoint) startup_state = self._register() # Validators version 1.1 send startup info with the registration # response; newer versions will send an activation message with the # startup info if startup_state is None: startup_state = self._wait_until_active() self._updates = Queue() driver_thread = Thread( target=self._driver_loop) driver_thread.start() try: self._engine.start( self._updates, ZmqService( stream=self._stream, timeout=SERVICE_TIMEOUT), startup_state) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught engine exception") self.stop() driver_thread.join() def _driver_loop(self): try: future = self._stream.receive() while True: if self._exit: self._engine.stop() break try: message = future.result(1) future = self._stream.receive() except concurrent.futures.TimeoutError: continue result = self._process(message) self._updates.put(result) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught driver exception") def stop(self): self._exit = True self._engine.stop() self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ).SerializeToString() while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if ( response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY ): continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: if ( response.HasField('chain_head') and response.HasField('local_peer_info') ): return StartupState( response.chain_head, response.peers, response.local_peer_info) return None raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _wait_until_active(self): future = self._stream.receive() while True: try: message = future.result(1) except concurrent.futures.TimeoutError: continue if ( message.message_type == Message.CONSENSUS_NOTIFY_ENGINE_ACTIVATED ): notification = \ consensus_pb2.ConsensusNotifyEngineActivated() notification.ParseFromString(message.content) startup_state = StartupState( notification.chain_head, notification.peers, notification.local_peer_info) LOGGER.info( 'Received activation message with startup state: %s', startup_state) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck() .SerializeToString()) return startup_state LOGGER.warning('Received message type %s while waiting for \ activation message', message.message_type) future = self._stream.receive() def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) header = consensus_pb2.ConsensusPeerMessageHeader() header.ParseFromString(notification.message.header) peer_message = PeerMessage( header=header, header_bytes=notification.message.header, header_signature=notification.message.header_signature, content=notification.message.content) data = peer_message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_ENGINE_DEACTIVATED: self.stop() data = None else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
class Subscriber(object): """Creates an object that can subscribe to state delta events using the Sawtooth SDK's Stream class. Handler functions can be added prior to subscribing, and each will be called on each delta event received. """ def __init__(self, validator_url): LOGGER.info('Connecting to validator: %s', validator_url) self._stream = Stream(validator_url) self._event_handlers = [] self._is_active = False def add_handler(self, handler): """Adds a handler which will be passed state delta events when they occur. Note that this event is mutable. """ self._event_handlers.append(handler) def clear_handlers(self): """Clears any delta handlers. """ self._event_handlers = [] def start(self, known_ids=None): """Subscribes to state delta events, and then waits to receive deltas. Sends any events received to delta handlers. """ if not known_ids: known_ids = [NULL_BLOCK_ID] self._stream.wait_for_ready() LOGGER.debug('Subscribing to state delta events') block_sub = EventSubscription(event_type='sawtooth/block-commit') delta_sub = EventSubscription( event_type='sawtooth/state-delta', filters=[ EventFilter(key='address', match_string='^{}.*'.format(NAMESPACE), filter_type=EventFilter.REGEX_ANY) ]) request = ClientEventsSubscribeRequest( last_known_block_ids=known_ids, subscriptions=[block_sub, delta_sub]) response_future = self._stream.send( Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()) response = ClientEventsSubscribeResponse() response.ParseFromString(response_future.result().content) # Forked all the way back to genesis, restart with no known_ids if (response.status == ClientEventsSubscribeResponse.UNKNOWN_BLOCK and known_ids): return self.start() if response.status != ClientEventsSubscribeResponse.OK: raise RuntimeError('Subscription failed with status: {}'.format( ClientEventsSubscribeResponse.Status.Name(response.status))) self._is_active = True LOGGER.debug('Successfully subscribed to state delta events') while self._is_active: message_future = self._stream.receive() event_list = EventList() event_list.ParseFromString(message_future.result().content) for handler in self._event_handlers: handler(event_list.events) def stop(self): """Stops the Subscriber, unsubscribing from state delta events and closing the the stream's connection. """ self._is_active = False LOGGER.debug('Unsubscribing from state delta events') request = ClientEventsUnsubscribeRequest() response_future = self._stream.send( Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()) response = ClientEventsUnsubscribeResponse() response.ParseFromString(response_future.result().content) if response.status != ClientEventsUnsubscribeResponse.OK: LOGGER.warning( 'Failed to unsubscribe with status: %s', ClientEventsUnsubscribeResponse.Status.Name(response.status)) self._stream.close()
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): self._stream = Stream(endpoint) startup_state = self._register() self._updates = Queue() driver_thread = Thread( target=self._driver_loop) driver_thread.start() try: self._engine.start( self._updates, ZmqService( stream=self._stream, timeout=SERVICE_TIMEOUT, name=self._engine.name(), version=self._engine.version()), startup_state) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught engine exception") self.stop() driver_thread.join() def _driver_loop(self): try: future = self._stream.receive() while True: if self._exit: self._engine.stop() break try: message = future.result(1) future = self._stream.receive() except concurrent.futures.TimeoutError: continue result = self._process(message) self._updates.put(result) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught driver exception") def stop(self): self._exit = True self._engine.stop() self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ).SerializeToString() while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if ( response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY ): continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: return StartupState( response.chain_head, response.peers, response.local_peer_info) raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) data = notification.message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
def listen_to_events(delta_filters=None): '''Listen to all state-delta events from the attestation TF.''' trustQueryHits = 0 trustQueryMisses = 0 # Subscribe to events evidence_submission_subscription = events_pb2.EventSubscription( event_type="attestation/evidence_submission", filters=delta_filters) evidence_deletion_subscription = events_pb2.EventSubscription( event_type="attestation/evidence_deletion", filters=delta_filters) trust_path_subscription = events_pb2.EventSubscription( event_type="attestation/trustpath", filters=delta_filters) trust_entry_subscription = events_pb2.EventSubscription( event_type="attestation/entrypoint", filters=delta_filters) request = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=[evidence_submission_subscription, evidence_deletion_subscription, trust_path_subscription, trust_entry_subscription]) ''' block_commit_subscription = events_pb2.EventSubscription( event_type="sawtooth/block-commit") state_delta_subscription = events_pb2.EventSubscription( event_type="sawtooth/state-delta", filters=delta_filters) request = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=[block_commit_subscription, state_delta_subscription]) ''' # Send the subscription request stream = Stream(DEFAULT_VALIDATOR_URL) msg = stream.send(message_type=Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, content=request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE # Parse the subscription response response = client_event_pb2.ClientEventsSubscribeResponse() response.ParseFromString(msg.content) assert response.status == \ client_event_pb2.ClientEventsSubscribeResponse.OK # Listen for events in an infinite loop print("Listening to events.") lastevent = None while True: msg = stream.receive().result() assert msg.message_type == Message.CLIENT_EVENTS # Parse the response event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) print("Received the following events: ----------") for event in event_list.events: if event == lastevent: continue else: lastevent = event print(event) if (event.event_type == "attestation/evidence_submission"): vrf = event.attributes[0].value prv = event.attributes[1].value writeEdgeData(vrf, prv) elif (event.event_type == "attestation/evidence_deletion"): vrf = event.attributes[0].value prv = event.attributes[1].value deleteEdgeData(vrf, prv) elif (event.event_type == "attestation/trustpath"): trustQueryHits +=1 elif (event.event_type == "attestation/entrypoint"): trustQueryMisses +=1 # Unsubscribe from events request = client_event_pb2.ClientEventsUnsubscribeRequest() msg = stream.send(Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() assert msg.message_type == Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE # Parse the unsubscribe response response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(msg.content) assert response.status == \ client_event_pb2.ClientEventsUnsubscribeResponse.OK
class TransactionProcessor(object): def __init__(self, url): self._stream = Stream(url) self._url = url self._handlers = [] @property def zmq_id(self): return self._stream.zmq_id def add_handler(self, handler): """Add a transaction family handler :param handler: """ self._handlers.append(handler) def _matches(self, handler, header): return header.family_name == handler.family_name \ and header.family_version in handler.family_versions \ and header.payload_encoding in handler.encodings def _find_handler(self, header): """Find a handler for a particular (family_name, family_versions, payload_encoding) :param header transaction_pb2.TransactionHeader: :return: handler """ try: return next(handler for handler in self._handlers if self._matches(handler, header)) except StopIteration: LOGGER.debug("Missing handler for header: %s", header) return None def _register_requests(self): """Returns all of the TpRegisterRequests for handlers :return (list): list of TpRegisterRequests """ return itertools.chain.from_iterable( # flattens the nested list [[ TpRegisterRequest(family=n, version=v, encoding=e, namespaces=h.namespaces) for n, v, e in itertools.product( [h.family_name], h.family_versions, h.encodings) ] for h in self._handlers]) def _unregister_request(self): """Returns a single TP_UnregisterRequest that requests that the validator stop sending transactions for previously registered handlers. :return (processor_pb2.TpUnregisterRequest): """ return TpUnregisterRequest() def _process(self, msg): if msg.message_type != Message.TP_PROCESS_REQUEST: LOGGER.debug( "Transaction Processor recieved invalid message type. " "Message type should be TP_PROCESS_REQUEST," " but is %s", Message.MessageType.Name(msg.message_type)) return request = TpProcessRequest() request.ParseFromString(msg.content) state = State(self._stream, request.context_id) header = TransactionHeader() header.ParseFromString(request.header) try: if not self._stream.is_ready(): raise ValidatorConnectionError() handler = self._find_handler(header) if handler is None: return handler.apply(request, state) self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.OK).SerializeToString()) except InvalidTransaction as it: LOGGER.warning("Invalid Transaction %s", it) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(it), extended_data=it.extended_data).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) except InternalError as ie: LOGGER.warning("internal error: %s", ie) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INTERNAL_ERROR, message=str(ie), extended_data=ie.extended_data).SerializeToString()) except ValidatorConnectionError as vce: # Same as the prior except block, but an internal error has # happened, but because of the disconnect the validator # probably doesn't care about the response. LOGGER.warning("during internal error response: %s", vce) except ValidatorConnectionError as vce: # Somewhere within handler.apply a future resolved with an # error status that the validator has disconnected. There is # nothing left to do but reconnect. LOGGER.warning( "during handler.apply a future was resolved " "with error status: %s", vce) def _process_future(self, future, timeout=None, sigint=False): try: msg = future.result(timeout) except CancelledError: # This error is raised when Task.cancel is called on # disconnect from the validator in stream.py, for # this future. return if msg is RECONNECT_EVENT: if sigint is False: LOGGER.info("reregistering with validator") self._stream.wait_for_ready() self._register() else: LOGGER.debug('received message of type: %s', Message.MessageType.Name(msg.message_type)) if msg.message_type == Message.TP_PING: self._stream.send_back( message_type=Message.TP_PING_RESPONSE, correlation_id=msg.correlation_id, content=TpPingResponse( status=TpPingResponse.OK).SerializeToString()) return self._process(msg) def _register(self): futures = [] for message in self._register_requests(): self._stream.wait_for_ready() future = self._stream.send( message_type=Message.TP_REGISTER_REQUEST, content=message.SerializeToString()) futures.append(future) for future in futures: resp = TpRegisterResponse() try: resp.ParseFromString(future.result().content) LOGGER.info("register attempt: %s", TpRegisterResponse.Status.Name(resp.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on registration: %s", vce) def _unregister(self): message = self._unregister_request() self._stream.wait_for_ready() future = self._stream.send(message_type=Message.TP_UNREGISTER_REQUEST, content=message.SerializeToString()) response = TpUnregisterResponse() try: response.ParseFromString(future.result(1).content) LOGGER.info("unregister attempt: %s", TpUnregisterResponse.Status.Name(response.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on unregistration: %s", vce) def start(self): fut = None try: self._register() while True: # During long running processing this # is where the transaction processor will # spend most of its time fut = self._stream.receive() self._process_future(fut) except KeyboardInterrupt: try: # tell the validator to not send any more messages self._unregister() while True: if fut is not None: # process futures as long as the tp has them, # if the TP_PROCESS_REQUEST doesn't come from # zeromq->asyncio in 1 second raise a # concurrent.futures.TimeOutError and be done. self._process_future(fut, 1, sigint=True) fut = self._stream.receive() except concurrent.futures.TimeoutError: # Where the tp will usually exit after # a KeyboardInterrupt. Caused by the 1 second # timeout in _process_future. pass except FutureTimeoutError: # If the validator is not able to respond to the # unregister request, exit. pass def stop(self): self._stream.close()
class Subscriber(object): """Creates an object that can subscribe to state delta events using the Sawtooth SDK's Stream class. Handler functions can be added prior to subscribing, and each will be called on each delta event received. """ def __init__(self, validator_url): LOGGER.info('Connecting to validator: %s', validator_url) self._stream = Stream(validator_url) self._event_handlers = [] self._is_active = False def add_handler(self, handler): """Adds a handler which will be passed state delta events when they occur. Note that this event is mutable. """ self._event_handlers.append(handler) def clear_handlers(self): """Clears any delta handlers. """ self._event_handlers = [] def start(self, known_ids=None): """Subscribes to state delta events, and then waits to receive deltas. Sends any events received to delta handlers. """ if not known_ids: known_ids = [NULL_BLOCK_ID] self._stream.wait_for_ready() LOGGER.debug('Subscribing to state delta events') block_sub = EventSubscription(event_type='sawtooth/block-commit') delta_sub = EventSubscription( event_type='sawtooth/state-delta', filters=[EventFilter( key='address', match_string='^{}.*'.format(NAMESPACE), filter_type=EventFilter.REGEX_ANY)]) request = ClientEventsSubscribeRequest( last_known_block_ids=known_ids, subscriptions=[block_sub, delta_sub]) response_future = self._stream.send( Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()) response = ClientEventsSubscribeResponse() response.ParseFromString(response_future.result().content) # Forked all the way back to genesis, restart with no known_ids if (response.status == ClientEventsSubscribeResponse.UNKNOWN_BLOCK and known_ids): self.start() if response.status != ClientEventsSubscribeResponse.OK: raise RuntimeError( 'Subscription failed with status: {}'.format( ClientEventsSubscribeResponse.Status.Name( response.status))) self._is_active = True LOGGER.debug('Successfully subscribed to state delta events') while self._is_active: message_future = self._stream.receive() event_list = EventList() event_list.ParseFromString(message_future.result().content) for handler in self._event_handlers: handler(event_list.events) def stop(self): """Stops the Subscriber, unsubscribing from state delta events and closing the the stream's connection. """ self._is_active = False LOGGER.debug('Unsubscribing from state delta events') request = ClientEventsUnsubscribeRequest() response_future = self._stream.send( Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()) response = ClientEventsUnsubscribeResponse() response.ParseFromString(response_future.result().content) if response.status != ClientEventsUnsubscribeResponse.OK: LOGGER.warning( 'Failed to unsubscribe with status: %s', ClientEventsUnsubscribeResponse.Status.Name(response.status)) self._stream.close()