def do_load(args): with open(args.filename, mode='rb') as fd: batches = batch_pb2.BatchList() batches.ParseFromString(fd.read()) stream = Stream(args.url) futures = [] start = time.time() for batch_list in _split_batch_list(batches): future = stream.send( message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST, content=batch_list.SerializeToString()) futures.append(future) for future in futures: result = future.result() try: assert result.message_type == Message.CLIENT_BATCH_SUBMIT_RESPONSE except ValidatorConnectionError as vce: LOGGER.warning("the future resolved to %s", vce) stop = time.time() print("batches: {} batch/sec: {}".format( str(len(batches.batches)), len(batches.batches) / (stop - start))) stream.close()
def main(): stream = None try: opts = parse_args(sys.argv[1:]) stream = Stream(opts.stream_url) log_config = get_log_config(filename="rest_api_log_config.toml") if log_config is not None: log_configuration(log_config=log_config) else: log_dir = get_log_dir() log_configuration(log_dir=log_dir, name="sawtooth_rest_api") init_console_logging(verbose_level=opts.verbose) start_rest_api( opts.host, int(opts.port), stream, int(opts.timeout)) # pylint: disable=broad-except except Exception as e: print("Error: {}".format(e), file=sys.stderr) sys.exit(1) finally: if stream is not None: stream.close()
def do_subscribe(opts): opts_config = SubscriberConfig( connect=opts.connect, database_name=opts.database_name, database_host=opts.database_host, database_port=opts.database_port, database_user=opts.database_user, database_password=opts.database_password) subscriber_config = load_subscriber_config(opts_config) subscriber = None stream = None connection = None # pylint: disable=broad-except try: url = None if "tcp://" not in subscriber_config.connect: url = "tcp://" + subscriber_config.connect else: url = subscriber_config.connect stream = Stream(url) connection = psycopg2.connect( dbname=subscriber_config.database_name, host=subscriber_config.database_host, port=subscriber_config.database_port, user=subscriber_config.database_user, password=subscriber_config.database_password) subscriber = Subscriber(stream, connection) log_config = get_log_config( filename="supplychain_sds_log_config.toml") if log_config is not None: log_configuration(log_config=log_config) else: log_dir = get_log_dir() # use the stream zmq identity for filename log_configuration( log_dir=log_dir, name="supplychain-sds-" + str(stream.zmq_id)[2:-1]) subscriber.start() except KeyboardInterrupt: pass except Exception as e: print('Error: {}'.format(e), file=sys.stderr) finally: if subscriber is not None: subscriber.shutdown() if stream is not None: stream.close() if connection is not None: connection.close()
def main(): stream = None try: opts = parse_args(sys.argv[1:]) stream = Stream(opts.stream_url) start_rest_api( opts.host, int(opts.port), stream, int(opts.timeout)) # pylint: disable=broad-except except Exception as e: print("Error: {}".format(e), file=sys.stderr) sys.exit(1) finally: if stream is not None: stream.close()
def main(): stream = None try: opts = parse_args(sys.argv[1:]) opts_config = RestApiConfig(bind=opts.bind, connect=opts.connect, timeout=opts.timeout) rest_api_config = load_rest_api_config(opts_config) if "tcp://" not in rest_api_config.connect: stream = Stream("tcp://" + rest_api_config.connect) else: stream = Stream(rest_api_config.connect) log_config = get_log_config(filename="rest_api_log_config.toml") if log_config is not None: log_configuration(log_config=log_config) else: log_dir = get_log_dir() log_configuration(log_dir=log_dir, name="sawtooth_rest_api") init_console_logging(verbose_level=opts.verbose) try: host, port = rest_api_config.bind[0].split(":") port = int(port) except ValueError as e: print("Unable to parse binding {}: Must be in the format" " host:port".format(rest_api_config.bind[0])) sys.exit(1) start_rest_api(host, port, stream, int(rest_api_config.timeout)) # pylint: disable=broad-except except Exception as e: print("Error: {}".format(e), file=sys.stderr) sys.exit(1) finally: if stream is not None: stream.close()
class Subscriber(object): """Creates an object that can subscribe to state delta events using the Sawtooth SDK's Stream class. Handler functions can be added prior to subscribing, and each will be called on each delta event received. """ def __init__(self, validator_url): LOGGER.info("Connecting to validator: %s", validator_url) self._stream = Stream(validator_url) self._delta_handlers = [] self._is_active = False def add_handler(self, event_handler): """Adds a handler which will be passed state delta events when they occur. Note that this event is mutable. """ self._delta_handlers.append(event_handler) def clear_handlers(self): """Clears any delta handlers. """ self._delta_handlers = [] def start(self, known_ids=None): """Subscribes to state delta events, and then waits to receive deltas. Sends any events received to delta handlers. """ self._stream.wait_for_ready() LOGGER.debug("Subscribing to client state events") request = client_event_pb2.ClientEventsSubscribeRequest( last_known_block_ids=known_ids, subscriptions=[ events_pb2.EventSubscription( event_type="sawtooth/block-commit"), events_pb2.EventSubscription( event_type="sawtooth/state-delta", filters=[ events_pb2.EventFilter( key="address", match_string="^" + addresser.NAMESPACE + ".*", filter_type=events_pb2.EventFilter.REGEX_ANY, ) ], ), ], ) response_future = self._stream.send( Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()) response = client_event_pb2.ClientEventsSubscribeResponse() response.ParseFromString(response_future.result().content) # Forked all the way back to genesis, restart with no known_ids if (known_ids and response.status == client_event_pb2.ClientEventsSubscribeResponse.UNKNOWN_BLOCK): return self.start() if response.status != client_event_pb2.ClientEventsSubscribeResponse.OK: raise RuntimeError("Subscription failed with status: {}".format( client_event_pb2.ClientEventsSubscribeResponse.Status.Name( response.status))) self._is_active = True LOGGER.debug("Successfully subscribed to state delta events") while self._is_active: message_future = self._stream.receive() msg = message_future.result() if msg.message_type == Message.CLIENT_EVENTS: event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = list(event_list.events) event = StateDeltaEvent(events) delta_count = len(event.state_changes) if delta_count > 0: LOGGER.debug("Received %d deltas for block: %s", delta_count, event.block_id) for handler in self._delta_handlers: handler(event) def stop(self): """Stops the Subscriber, unsubscribing from state delta events and closing the the stream's connection. """ self._is_active = False LOGGER.debug("Unsubscribing from client events") request = client_event_pb2.ClientEventsUnsubscribeResponse() response_future = self._stream.send( Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()) response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(response_future.result().content) if response.status != client_event_pb2.ClientEventsUnsubscribeResponse.OK: LOGGER.warning( "Failed to unsubscribe with status: %s", client_event_pb2.ClientEventsUnsubscribeResponse.Status.Name( response.status), ) self._stream.close()
class TestEventsAndReceipts(unittest.TestCase): def test_subscribe_and_unsubscribe(self): """Tests that a client can subscribe and unsubscribe from events.""" response = self._subscribe() self.assert_subscribe_response(response) response = self._unsubscribe() self.assert_unsubscribe_response(response) def test_block_commit_event_received(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() for i in range(1, 5): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self.assertEqual(msg.message_type, validator_pb2.Message.CLIENT_EVENTS) event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 1) self.assert_block_commit_event(events[0], i) self._unsubscribe() def test_get_events(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self._unsubscribe() event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events block_commit_event = events[0] block_id = list( filter(lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list( filter(lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value response = self._get_events( block_id, [events_pb2.EventSubscription(event_type="sawtooth/block-commit")]) events = self.assert_events_get_response(response) self.assert_block_commit_event(events[0], block_num) def test_catchup(self): """Tests that a subscriber correctly receives catchup events.""" self._subscribe() blocks = [] for i in range(4): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events block_commit_event = events[0] block_id = list( filter(lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list( filter(lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value blocks.append((block_num, block_id)) self._unsubscribe() self.assert_subscribe_response( self._subscribe(last_known_block_ids=[blocks[0][1]])) LOGGER.warning("Waiting for catchup events") for i in range(3): msg = self.stream.receive().result() LOGGER.warning("Got catchup events: ") event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 1) block_commit_event = events[0] block_id = list( filter(lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list( filter(lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value self.assertEqual((block_num, block_id), blocks[i + 1]) self._unsubscribe() def test_receipt_stored(self): """Tests that receipts are stored successfully when a block is committed.""" self._subscribe() n = self.batch_submitter.submit_next_batch() response = self._get_receipt(n) receipts = self.assert_receipt_get_response(response) state_change = receipts[0].state_changes[0] self.assertEqual(state_change.type, transaction_receipt_pb2.StateChange.SET) self.assertEqual(state_change.value, cbor.dumps({str(n): 0})) self.assertEqual(state_change.address, make_intkey_address(str(n))) self._unsubscribe() @classmethod def setUpClass(cls): cls.batch_submitter = BatchSubmitter(WAIT) def setUp(self): self.url = "tcp://validator:4004" self.stream = Stream(self.url) def tearDown(self): if self.stream is not None: self.stream.close() def _get_receipt(self, n): txn_id = \ self.batch_submitter.batches[n].transactions[0].header_signature request = client_receipt_pb2.ClientReceiptGetRequest( transaction_ids=[txn_id]) response = self.stream.send( validator_pb2.Message.CLIENT_RECEIPT_GET_REQUEST, request.SerializeToString()).result() return response def _get_events(self, block_id, subscriptions): request = client_event_pb2.ClientEventsGetRequest( block_ids=[block_id], subscriptions=subscriptions) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_GET_REQUEST, request.SerializeToString()).result() return response def _subscribe(self, subscriptions=None, last_known_block_ids=None): if subscriptions is None: subscriptions = [ events_pb2.EventSubscription( event_type="sawtooth/block-commit"), ] if last_known_block_ids is None: last_known_block_ids = [] request = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=subscriptions, last_known_block_ids=last_known_block_ids) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def _unsubscribe(self): request = client_event_pb2.ClientEventsUnsubscribeRequest() response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def assert_block_commit_event(self, event, block_num): self.assertEqual(event.event_type, "sawtooth/block-commit") self.assertTrue( all([ any(attribute.key == "block_id" for attribute in event.attributes), any(attribute.key == "block_num" for attribute in event.attributes), any(attribute.key == "previous_block_id" for attribute in event.attributes), any(attribute.key == "state_root_hash" for attribute in event.attributes), ])) for attribute in event.attributes: if attribute.key == "block_num": self.assertEqual(attribute.value, str(block_num)) def assert_receipt_get_response(self, msg): self.assertEqual(msg.message_type, validator_pb2.Message.CLIENT_RECEIPT_GET_RESPONSE) receipt_response = client_receipt_pb2.ClientReceiptGetResponse() receipt_response.ParseFromString(msg.content) self.assertEqual(receipt_response.status, client_receipt_pb2.ClientReceiptGetResponse.OK) return receipt_response.receipts def assert_events_get_response(self, msg): self.assertEqual(msg.message_type, validator_pb2.Message.CLIENT_EVENTS_GET_RESPONSE) events_response = client_event_pb2.ClientEventsGetResponse() events_response.ParseFromString(msg.content) self.assertEqual(events_response.status, client_event_pb2.ClientEventsGetResponse.OK) return events_response.events def assert_subscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE) subscription_response = client_event_pb2.ClientEventsSubscribeResponse( ) subscription_response.ParseFromString(msg.content) self.assertEqual(subscription_response.status, client_event_pb2.ClientEventsSubscribeResponse.OK) def assert_unsubscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE) subscription_response = client_event_pb2.ClientEventsUnsubscribeResponse( ) subscription_response.ParseFromString(msg.content) self.assertEqual(subscription_response.status, client_event_pb2.ClientEventsUnsubscribeResponse.OK)
class TransactionProcessor: """TransactionProcessor is a generic class for communicating with a validator and routing transaction processing requests to a registered handler. It uses ZMQ and channels to handle requests concurrently. """ def __init__(self, url): """ Args: url (string): The URL of the validator """ self._stream = Stream(url) self._url = url self._handlers = [] @property def zmq_id(self): return self._stream.zmq_id def add_handler(self, handler): """Adds a transaction family handler Args: handler (TransactionHandler): the handler to be added """ self._handlers.append(handler) def _matches(self, handler, header): return header.family_name == handler.family_name \ and header.family_version in handler.family_versions def _find_handler(self, header): """Find a handler for a particular (family_name, family_versions) :param header transaction_pb2.TransactionHeader: :return: handler """ try: return next( handler for handler in self._handlers if self._matches(handler, header)) except StopIteration: LOGGER.debug("Missing handler for header: %s", header) return None def _register_requests(self): """Returns all of the TpRegisterRequests for handlers :return (list): list of TpRegisterRequests """ return itertools.chain.from_iterable( # flattens the nested list [ [TpRegisterRequest( family=n, version=v, namespaces=h.namespaces) for n, v in itertools.product( [h.family_name], h.family_versions,)] for h in self._handlers]) def _unregister_request(self): """Returns a single TP_UnregisterRequest that requests that the validator stop sending transactions for previously registered handlers. :return (processor_pb2.TpUnregisterRequest): """ return TpUnregisterRequest() def _process(self, msg): if msg.message_type != Message.TP_PROCESS_REQUEST: LOGGER.debug( "Transaction Processor recieved invalid message type. " "Message type should be TP_PROCESS_REQUEST," " but is %s", Message.MessageType.Name(msg.message_type)) return request = TpProcessRequest() request.ParseFromString(msg.content) state = Context(self._stream, request.context_id) header = request.header try: if not self._stream.is_ready(): raise ValidatorConnectionError() handler = self._find_handler(header) if handler is None: return handler.apply(request, state) self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.OK ).SerializeToString()) except InvalidTransaction as it: LOGGER.warning("Invalid Transaction %s", it) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(it), extended_data=it.extended_data ).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) except InternalError as ie: LOGGER.warning("internal error: %s", ie) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INTERNAL_ERROR, message=str(ie), extended_data=ie.extended_data ).SerializeToString()) except ValidatorConnectionError as vce: # Same as the prior except block, but an internal error has # happened, but because of the disconnect the validator # probably doesn't care about the response. LOGGER.warning("during internal error response: %s", vce) except ValidatorConnectionError as vce: # Somewhere within handler.apply a future resolved with an # error status that the validator has disconnected. There is # nothing left to do but reconnect. LOGGER.warning("during handler.apply a future was resolved " "with error status: %s", vce) except AuthorizationException as ae: LOGGER.warning("AuthorizationException: %s", ae) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(ae), ).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) def _process_future(self, future, timeout=None, sigint=False): try: msg = future.result(timeout) except CancelledError: # This error is raised when Task.cancel is called on # disconnect from the validator in stream.py, for # this future. return if msg is RECONNECT_EVENT: if sigint is False: LOGGER.info("reregistering with validator") self._stream.wait_for_ready() self._register() else: LOGGER.debug( 'received message of type: %s', Message.MessageType.Name(msg.message_type)) if msg.message_type == Message.PING_REQUEST: self._stream.send_back( message_type=Message.PING_RESPONSE, correlation_id=msg.correlation_id, content=PingResponse().SerializeToString()) return self._process(msg) def _register(self): futures = [] for message in self._register_requests(): self._stream.wait_for_ready() future = self._stream.send( message_type=Message.TP_REGISTER_REQUEST, content=message.SerializeToString()) futures.append(future) for future in futures: resp = TpRegisterResponse() try: resp.ParseFromString(future.result().content) LOGGER.info("register attempt: %s", TpRegisterResponse.Status.Name(resp.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on registration: %s", vce) def _unregister(self): message = self._unregister_request() self._stream.wait_for_ready() future = self._stream.send( message_type=Message.TP_UNREGISTER_REQUEST, content=message.SerializeToString()) response = TpUnregisterResponse() try: response.ParseFromString(future.result(1).content) LOGGER.info("unregister attempt: %s", TpUnregisterResponse.Status.Name(response.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on unregistration: %s", vce) def start(self): """Connects the transaction processor to a validator and starts listening for requests and routing them to an appropriate transaction handler. """ fut = None try: self._register() while True: # During long running processing this # is where the transaction processor will # spend most of its time fut = self._stream.receive() self._process_future(fut) except KeyboardInterrupt: try: # tell the validator to not send any more messages self._unregister() while True: if fut is not None: # process futures as long as the tp has them, # if the TP_PROCESS_REQUEST doesn't come from # zeromq->asyncio in 1 second raise a # concurrent.futures.TimeOutError and be done. self._process_future(fut, 1, sigint=True) fut = self._stream.receive() except concurrent.futures.TimeoutError: # Where the tp will usually exit after # a KeyboardInterrupt. Caused by the 1 second # timeout in _process_future. pass except FutureTimeoutError: # If the validator is not able to respond to the # unregister request, exit. pass def stop(self): """Closes the connection between the TransactionProcessor and the validator. """ self._stream.close()
class Subscriber(object): """Creates an object that can subscribe to state delta events using the Sawtooth SDK's Stream class. Handler functions can be added prior to subscribing, and each will be called on each delta event received. """ def __init__(self, validator_url): LOGGER.info('Connecting to validator: %s', validator_url) self._stream = Stream(validator_url) self._event_handlers = [] self._is_active = False def add_handler(self, handler): """Adds a handler which will be passed state delta events when they occur. Note that this event is mutable. """ self._event_handlers.append(handler) def clear_handlers(self): """Clears any delta handlers. """ self._event_handlers = [] def start(self, known_ids=None): """Subscribes to state delta events, and then waits to receive deltas. Sends any events received to delta handlers. """ if not known_ids: known_ids = [NULL_BLOCK_ID] self._stream.wait_for_ready() LOGGER.debug('Subscribing to state delta events') block_sub = EventSubscription(event_type='sawtooth/block-commit') delta_sub = EventSubscription( event_type='sawtooth/state-delta', filters=[ EventFilter(key='address', match_string='^{}.*'.format(NAMESPACE), filter_type=EventFilter.REGEX_ANY) ]) request = ClientEventsSubscribeRequest( last_known_block_ids=known_ids, subscriptions=[block_sub, delta_sub]) response_future = self._stream.send( Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()) response = ClientEventsSubscribeResponse() response.ParseFromString(response_future.result().content) # Forked all the way back to genesis, restart with no known_ids if (response.status == ClientEventsSubscribeResponse.UNKNOWN_BLOCK and known_ids): return self.start() if response.status != ClientEventsSubscribeResponse.OK: raise RuntimeError('Subscription failed with status: {}'.format( ClientEventsSubscribeResponse.Status.Name(response.status))) self._is_active = True LOGGER.debug('Successfully subscribed to state delta events') while self._is_active: message_future = self._stream.receive() event_list = EventList() event_list.ParseFromString(message_future.result().content) for handler in self._event_handlers: handler(event_list.events) def stop(self): """Stops the Subscriber, unsubscribing from state delta events and closing the the stream's connection. """ self._is_active = False LOGGER.debug('Unsubscribing from state delta events') request = ClientEventsUnsubscribeRequest() response_future = self._stream.send( Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()) response = ClientEventsUnsubscribeResponse() response.ParseFromString(response_future.result().content) if response.status != ClientEventsUnsubscribeResponse.OK: LOGGER.warning( 'Failed to unsubscribe with status: %s', ClientEventsUnsubscribeResponse.Status.Name(response.status)) self._stream.close()
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): self._stream = Stream(endpoint) startup_state = self._register() # Validators version 1.1 send startup info with the registration # response; newer versions will send an activation message with the # startup info if startup_state is None: startup_state = self._wait_until_active() self._updates = Queue() driver_thread = Thread( target=self._driver_loop) driver_thread.start() try: self._engine.start( self._updates, ZmqService( stream=self._stream, timeout=SERVICE_TIMEOUT), startup_state) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught engine exception") self.stop() driver_thread.join() def _driver_loop(self): try: future = self._stream.receive() while True: if self._exit: self._engine.stop() break try: message = future.result(1) future = self._stream.receive() except concurrent.futures.TimeoutError: continue try: result = self._process(message) # if message was a ping ignore if result[0] == Message.PING_REQUEST: continue self._updates.put(result) except exceptions.ReceiveError as err: LOGGER.warning("%s", err) continue except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught driver exception") def stop(self): self._exit = True self._engine.stop() self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ) for (name, version) in self._engine.additional_protocols(): protocol = request.additional_protocols.add() protocol.name = name protocol.version = version while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request.SerializeToString()) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if ( response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY ): continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: if ( response.HasField('chain_head') and response.HasField('local_peer_info') ): return StartupState( response.chain_head, response.peers, response.local_peer_info) return None raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _wait_until_active(self): future = self._stream.receive() while True: try: message = future.result(1) except concurrent.futures.TimeoutError: continue if ( message.message_type == Message.CONSENSUS_NOTIFY_ENGINE_ACTIVATED ): notification = \ consensus_pb2.ConsensusNotifyEngineActivated() notification.ParseFromString(message.content) startup_state = StartupState( notification.chain_head, notification.peers, notification.local_peer_info) LOGGER.info( 'Received activation message with startup state: %s', startup_state) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck() .SerializeToString()) return startup_state LOGGER.warning('Received message type %s while waiting for \ activation message', message.message_type) future = self._stream.receive() def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) header = consensus_pb2.ConsensusPeerMessageHeader() header.ParseFromString(notification.message.header) peer_message = PeerMessage( header=header, header_bytes=notification.message.header, header_signature=notification.message.header_signature, content=notification.message.content) data = peer_message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_ENGINE_DEACTIVATED: self.stop() data = None elif type_tag == Message.PING_REQUEST: data = None else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): LOGGER.debug('ZmqDriver: start endpoint=%s', endpoint) self._stream = Stream(endpoint) startup_state = self._register() self._updates = Queue() driver_thread = Thread(target=self._driver_loop) driver_thread.start() try: self._engine.start( self._updates, ZmqService(stream=self._stream, timeout=SERVICE_TIMEOUT, name=self._engine.name(), version=self._engine.version()), startup_state) except Exception as ex: # pylint: disable=broad-except LOGGER.exception("Uncaught engine exception(%s)", ex) self.stop() driver_thread.join() def _driver_loop(self): try: future = self._stream.receive() LOGGER.debug('ZmqDriver: _driver_loop future=%s', future) while True: if self._exit: self._engine.stop() break try: message = future.result(1) future = self._stream.receive() except concurrent.futures.TimeoutError: continue #LOGGER.debug('ZmqDriver: _driver_loop _process') result = self._process(message) self._updates.put(result) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught driver exception") def stop(self): self._exit = True self._engine.stop() self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ).SerializeToString() while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if (response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY): #LOGGER.debug('ZmqDriver:register NOT_READY: url=%s',self._stream._url) continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: LOGGER.debug('ZmqDriver:register DONE: url=%s', self._stream._url) return StartupState(response.chain_head, response.peers, response.local_peer_info, response.peering_mode) raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info, notification.status, notification.mode, notification.info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) data = notification.message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
class TestEventsAndReceipts(unittest.TestCase): def test_subscribe_and_unsubscribe(self): """Tests that a client can subscribe and unsubscribe from events.""" response = self._subscribe() self.assert_subscribe_response(response) response = self._unsubscribe() self.assert_unsubscribe_response(response) def test_subscribe_and_unsubscribe_with_catch_up(self): """Tests that a client can subscribe and unsubscribe from events.""" response = self._subscribe( last_known_block_ids=[NULL_BLOCK_IDENTIFIER]) self.assert_subscribe_response(response) # Ensure that it receives the genesis block msg = self.stream.receive().result() self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS) event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 2) self.assert_block_commit_event(events[0], 0) self.assert_state_event(events[1], '000000') response = self._unsubscribe() self.assert_unsubscribe_response(response) def test_block_commit_event_received(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() for i in range(1, 5): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS) event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 2) self.assert_block_commit_event(events[0], i) self.assert_state_event(events[1], INTKEY_ADDRESS_PREFIX) self._unsubscribe() def test_get_events(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self._unsubscribe() event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events block_commit_event = events[0] block_id = list(filter( lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list(filter( lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value response = self._get_events( block_id, [events_pb2.EventSubscription(event_type="sawtooth/block-commit")]) events = self.assert_events_get_response(response) self.assert_block_commit_event(events[0], block_num) def test_catchup(self): """Tests that a subscriber correctly receives catchup events.""" self._subscribe() blocks = [] for i in range(4): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events block_commit_event = events[0] block_id = list(filter( lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list(filter( lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value blocks.append((block_num, block_id)) self._unsubscribe() self.assert_subscribe_response( self._subscribe(last_known_block_ids=[blocks[0][1]])) LOGGER.warning("Waiting for catchup events") for i in range(3): msg = self.stream.receive().result() LOGGER.warning("Got catchup events: ") event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 2) block_commit_event = events[0] block_id = list(filter( lambda attr: attr.key == "block_id", block_commit_event.attributes))[0].value block_num = list(filter( lambda attr: attr.key == "block_num", block_commit_event.attributes))[0].value self.assertEqual((block_num, block_id), blocks[i + 1]) self._unsubscribe() def test_receipt_stored(self): """Tests that receipts are stored successfully when a block is committed.""" self._subscribe() n = self.batch_submitter.submit_next_batch() response = self._get_receipt(n) receipts = self.assert_receipt_get_response(response) state_change = receipts[0].state_changes[0] self.assertEqual( state_change.type, transaction_receipt_pb2.StateChange.SET) self.assertEqual( state_change.value, cbor.dumps({str(n): 0})) self.assertEqual( state_change.address, make_intkey_address(str(n))) self._unsubscribe() @classmethod def setUpClass(cls): wait_for_rest_apis(['rest-api:8008']) cls.batch_submitter = BatchSubmitter(WAIT) def setUp(self): self.url = "tcp://validator:4004" self.stream = Stream(self.url) def tearDown(self): if self.stream is not None: self.stream.close() def _get_receipt(self, num): txn_id = \ self.batch_submitter.batches[num].transactions[0].header_signature request = client_receipt_pb2.ClientReceiptGetRequest( transaction_ids=[txn_id]) response = self.stream.send( validator_pb2.Message.CLIENT_RECEIPT_GET_REQUEST, request.SerializeToString()).result() return response def _get_events(self, block_id, subscriptions): request = client_event_pb2.ClientEventsGetRequest( block_ids=[block_id], subscriptions=subscriptions) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_GET_REQUEST, request.SerializeToString()).result() return response def _subscribe(self, subscriptions=None, last_known_block_ids=None): if subscriptions is None: subscriptions = [ events_pb2.EventSubscription( event_type="sawtooth/block-commit"), # Subscribe to the settings state events, to test genesis # catch-up. events_pb2.EventSubscription( event_type="sawtooth/state-delta", filters=[events_pb2.EventFilter( key='address', match_string='000000.*', filter_type=events_pb2.EventFilter.REGEX_ANY)]), # Subscribe to the intkey state events, to test additional # events. events_pb2.EventSubscription( event_type="sawtooth/state-delta", filters=[events_pb2.EventFilter( key='address', match_string='{}.*'.format(INTKEY_ADDRESS_PREFIX), filter_type=events_pb2.EventFilter.REGEX_ANY)]), ] if last_known_block_ids is None: last_known_block_ids = [] request = client_event_pb2.ClientEventsSubscribeRequest( subscriptions=subscriptions, last_known_block_ids=last_known_block_ids) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def _unsubscribe(self): request = client_event_pb2.ClientEventsUnsubscribeRequest() response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def assert_block_commit_event(self, event, block_num): self.assertEqual(event.event_type, "sawtooth/block-commit") self.assertTrue( all([ any(attribute.key == "block_id" for attribute in event.attributes), any(attribute.key == "block_num" for attribute in event.attributes), any(attribute.key == "previous_block_id" for attribute in event.attributes), any(attribute.key == "state_root_hash" for attribute in event.attributes), ])) for attribute in event.attributes: if attribute.key == "block_num": self.assertEqual(attribute.value, str(block_num)) def assert_receipt_get_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_RECEIPT_GET_RESPONSE) receipt_response = client_receipt_pb2.ClientReceiptGetResponse() receipt_response.ParseFromString(msg.content) self.assertEqual( receipt_response.status, client_receipt_pb2.ClientReceiptGetResponse.OK) return receipt_response.receipts def assert_state_event(self, event, address_prefix): self.assertEqual(event.event_type, "sawtooth/state-delta") state_change_list = transaction_receipt_pb2.StateChangeList() state_change_list.ParseFromString(event.data) for change in state_change_list.state_changes: self.assertTrue(change.address.startswith(address_prefix)) def assert_events_get_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_GET_RESPONSE) events_response = client_event_pb2.ClientEventsGetResponse() events_response.ParseFromString(msg.content) self.assertEqual( events_response.status, client_event_pb2.ClientEventsGetResponse.OK) return events_response.events def assert_subscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE) response = client_event_pb2.ClientEventsSubscribeResponse() response.ParseFromString(msg.content) self.assertEqual( response.status, client_event_pb2.ClientEventsSubscribeResponse.OK) def assert_unsubscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE) response = client_event_pb2.ClientEventsUnsubscribeResponse() response.ParseFromString(msg.content) self.assertEqual( response.status, client_event_pb2.ClientEventsUnsubscribeResponse.OK)
class TestEventBroadcaster(unittest.TestCase): def test_subscribe_and_unsubscribe(self): """Tests that a client can subscribe and unsubscribe from events.""" response = self._subscribe() self.assert_subscribe_response(response) response = self._unsubscribe() self.assert_unsubscribe_response(response) def test_block_commit_event_received(self): """Tests that block commit events are properly received on block boundaries.""" self._subscribe() for i in range(1, 5): self.batch_submitter.submit_next_batch() msg = self.stream.receive().result() self.assertEqual(msg.message_type, validator_pb2.Message.CLIENT_EVENTS) event_list = events_pb2.EventList() event_list.ParseFromString(msg.content) events = event_list.events self.assertEqual(len(events), 1) self.assert_block_commit_event(events[0], i) @classmethod def setUpClass(cls): cls.batch_submitter = BatchSubmitter() def setUp(self): self.url = "tcp://validator:4004" self.stream = Stream(self.url) def tearDown(self): if self.stream is not None: self.stream.close() def _subscribe(self, subscriptions=None): if subscriptions is None: subscriptions = [ events_pb2.EventSubscription(event_type="block_commit"), ] request = events_pb2.ClientEventsSubscribeRequest( subscriptions=subscriptions) response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def _unsubscribe(self): request = events_pb2.ClientEventsUnsubscribeRequest() response = self.stream.send( validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()).result() return response def assert_block_commit_event(self, event, block_num): self.assertEqual(event.event_type, "block_commit") self.assertTrue( all([ any(attribute.key == "block_id" for attribute in event.attributes), any(attribute.key == "block_num" for attribute in event.attributes), any(attribute.key == "previous_block_id" for attribute in event.attributes), any(attribute.key == "state_root_hash" for attribute in event.attributes), ])) for attribute in event.attributes: if attribute.key == "block_num": self.assertEqual(attribute.value, str(block_num)) def assert_subscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_SUBSCRIBE_RESPONSE) subscription_response = events_pb2.ClientEventsSubscribeResponse() subscription_response.ParseFromString(msg.content) self.assertEqual(subscription_response.status, events_pb2.ClientEventsSubscribeResponse.OK) def assert_unsubscribe_response(self, msg): self.assertEqual( msg.message_type, validator_pb2.Message.CLIENT_EVENTS_UNSUBSCRIBE_RESPONSE) subscription_response = events_pb2.ClientEventsUnsubscribeResponse() subscription_response.ParseFromString(msg.content) self.assertEqual(subscription_response.status, events_pb2.ClientEventsUnsubscribeResponse.OK)
class Subscriber(object): """Creates an object that can subscribe to state delta events using the Sawtooth SDK's Stream class. Handler functions can be added prior to subscribing, and each will be called on each delta event received. """ def __init__(self, validator_url): LOGGER.info('Connecting to validator: %s', validator_url) self._stream = Stream(validator_url) self._delta_handlers = [] self._is_active = False def add_handler(self, event_handler): """Adds a handler which will be passed state delta events when they occur. Note that this event is mutable. """ self._delta_handlers.append(event_handler) def clear_handlers(self): """Clears any delta handlers. """ self._delta_handlers = [] def start(self, known_ids=None): """Subscribes to state delta events, and then waits to receive deltas. Sends any events received to delta handlers. """ self._stream.wait_for_ready() LOGGER.debug('Subscribing to state delta events') request = StateDeltaSubscribeRequest(last_known_block_ids=known_ids, address_prefixes=[NAMESPACE]) response_future = self._stream.send( Message.STATE_DELTA_SUBSCRIBE_REQUEST, request.SerializeToString()) response = StateDeltaSubscribeResponse() response.ParseFromString(response_future.result().content) # Forked all the way back to genesis, restart with no known_ids if (known_ids and response.status == StateDeltaSubscribeResponse.UNKNOWN_BLOCK): return self.start() if response.status != StateDeltaSubscribeResponse.OK: raise RuntimeError('Subscription failed with status: {}'.format( StateDeltaSubscribeResponse.Status.Name(response.status))) self._is_active = True LOGGER.debug('Successfully subscribed to state delta events') while self._is_active: message_future = self._stream.receive() event = StateDeltaEvent() event.ParseFromString(message_future.result().content) LOGGER.debug('Received deltas for block: %s', event.block_id) for handler in self._delta_handlers: handler(event) def stop(self): """Stops the Subscriber, unsubscribing from state delta events and closing the the stream's connection. """ self._is_active = False LOGGER.debug('Unsubscribing from state delta events') request = StateDeltaUnsubscribeRequest() response_future = self._stream.send( Message.STATE_DELTA_UNSUBSCRIBE_REQUEST, request.SerializeToString()) response = StateDeltaUnsubscribeResponse() response.ParseFromString(response_future.result().content) if response.status != StateDeltaUnsubscribeResponse.OK: LOGGER.warning( 'Failed to unsubscribe with status: %s', StateDeltaUnsubscribeResponse.Status.Name(response.status)) self._stream.close()
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): self._stream = Stream(endpoint) (chain_head, peers) = self._register() self._updates = Queue() engine_thread = Thread( target=self._engine.start, args=(self._updates, ZmqService(stream=self._stream, timeout=SERVICE_TIMEOUT, name=self._engine.name(), version=self._engine.version()), chain_head, peers)) engine_thread.start() while True: if self._exit: self._engine.stop() engine_thread.join() break try: message = self._stream.receive().result(0.1) except concurrent.futures.TimeoutError: continue result = self._process(message) self._updates.put(result) def stop(self): self._exit = True self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ).SerializeToString() while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if (response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY): continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: return (response.chain_head, response.peers) raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) data = notification.message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
class TransactionProcessor: """TransactionProcessor is a generic class for communicating with a validator and routing transaction processing requests to a registered handler. It uses ZMQ and channels to handle requests concurrently. """ # This is the version used by SDK to match if validator supports feature # it requested during registration. It should only be incremented when # there are changes in TpRegisterRequest. Remember to sync this # information in validator if changed. # Note: SDK_PROTOCOL_VERSION is the highest version the SDK supports class _FeatureVersion(Enum): FEATURE_UNUSED = 0 FEATURE_CUSTOM_HEADER_STYLE = 1 SDK_PROTOCOL_VERSION = 1 def __init__(self, url): """ Args: url (string): The URL of the validator """ self._stream = Stream(url) self._url = url self._handlers = [] self._highest_sdk_feature_requested = \ self._FeatureVersion.FEATURE_UNUSED self._header_style = TpRegisterRequest.HEADER_STYLE_UNSET @property def zmq_id(self): return self._stream.zmq_id def add_handler(self, handler): """Adds a transaction family handler Args: handler (TransactionHandler): the handler to be added """ self._handlers.append(handler) def set_header_style(self, style): """Sets a flag to request the validator for custom transaction header style in TpProcessRequest. Args: style (TpProcessRequestHeaderStyle): enum value to set header style """ if self._FeatureVersion.FEATURE_CUSTOM_HEADER_STYLE.value > \ self._highest_sdk_feature_requested.value: self._highest_sdk_feature_requested = \ self._FeatureVersion.FEATURE_CUSTOM_HEADER_STYLE self._header_style = style def _matches(self, handler, header): return header.family_name == handler.family_name \ and header.family_version in handler.family_versions def _find_handler(self, header): """Find a handler for a particular (family_name, family_versions) :param header transaction_pb2.TransactionHeader: :return: handler """ try: return next(handler for handler in self._handlers if self._matches(handler, header)) except StopIteration: LOGGER.debug("Missing handler for header: %s", header) return None def _register_requests(self): """Returns all of the TpRegisterRequests for handlers :return (list): list of TpRegisterRequests """ return itertools.chain.from_iterable( # flattens the nested list [[ TpRegisterRequest( family=n, version=v, namespaces=h.namespaces, protocol_version=self._highest_sdk_feature_requested.value, request_header_style=self._header_style) for n, v in itertools.product( [h.family_name], h.family_versions, ) ] for h in self._handlers]) def _unregister_request(self): """Returns a single TP_UnregisterRequest that requests that the validator stop sending transactions for previously registered handlers. :return (processor_pb2.TpUnregisterRequest): """ return TpUnregisterRequest() def _process(self, msg): if msg.message_type != Message.TP_PROCESS_REQUEST: LOGGER.debug( "Transaction Processor recieved invalid message type. " "Message type should be TP_PROCESS_REQUEST," " but is %s", Message.MessageType.Name(msg.message_type)) return request = TpProcessRequest() request.ParseFromString(msg.content) state = Context(self._stream, request.context_id) if self._header_style == TpRegisterRequest.RAW: header = TransactionHeader() header.ParseFromString(request.header_bytes) else: header = request.header try: if not self._stream.is_ready(): raise ValidatorConnectionError() handler = self._find_handler(header) if handler is None: return handler.apply(request, state) self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.OK).SerializeToString()) except InvalidTransaction as it: LOGGER.warning("Invalid Transaction %s", it) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(it), extended_data=it.extended_data).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) except InternalError as ie: LOGGER.warning("internal error: %s", ie) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INTERNAL_ERROR, message=str(ie), extended_data=ie.extended_data).SerializeToString()) except ValidatorConnectionError as vce: # Same as the prior except block, but an internal error has # happened, but because of the disconnect the validator # probably doesn't care about the response. LOGGER.warning("during internal error response: %s", vce) except ValidatorConnectionError as vce: # Somewhere within handler.apply a future resolved with an # error status that the validator has disconnected. There is # nothing left to do but reconnect. LOGGER.warning( "during handler.apply a future was resolved " "with error status: %s", vce) except AuthorizationException as ae: LOGGER.warning("AuthorizationException: %s", ae) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(ae), ).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) def _process_future(self, future, timeout=None, sigint=False): try: msg = future.result(timeout) except CancelledError: # This error is raised when Task.cancel is called on # disconnect from the validator in stream.py, for # this future. return if msg is RECONNECT_EVENT: if sigint is False: LOGGER.info("reregistering with validator") self._stream.wait_for_ready() self._register() else: LOGGER.debug('received message of type: %s', Message.MessageType.Name(msg.message_type)) if msg.message_type == Message.PING_REQUEST: self._stream.send_back( message_type=Message.PING_RESPONSE, correlation_id=msg.correlation_id, content=PingResponse().SerializeToString()) return self._process(msg) def _register(self): futures = [] for message in self._register_requests(): self._stream.wait_for_ready() future = self._stream.send( message_type=Message.TP_REGISTER_REQUEST, content=message.SerializeToString()) futures.append(future) for future in futures: resp = TpRegisterResponse() try: resp.ParseFromString(future.result().content) if resp.protocol_version != \ self._highest_sdk_feature_requested.value: LOGGER.error( "Validator version %s does not support " "requested feature by SDK version %s. " "Unregistering with the validator.", str(resp.protocol_version), str(self._highest_sdk_feature_requested.value)) raise ValidatorVersionError() LOGGER.info("register attempt: %s", TpRegisterResponse.Status.Name(resp.status)) if resp.status == TpRegisterResponse.ERROR: raise RuntimeError("Transaction processor registration " "failed") except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on registration: %s", vce) def _unregister(self): message = self._unregister_request() self._stream.wait_for_ready() future = self._stream.send(message_type=Message.TP_UNREGISTER_REQUEST, content=message.SerializeToString()) response = TpUnregisterResponse() try: response.ParseFromString(future.result(1).content) LOGGER.info("unregister attempt: %s", TpUnregisterResponse.Status.Name(response.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on unregistration: %s", vce) def start(self): """Connects the transaction processor to a validator and starts listening for requests and routing them to an appropriate transaction handler. """ fut = None try: self._register() while True: # During long running processing this # is where the transaction processor will # spend most of its time fut = self._stream.receive() self._process_future(fut) except (KeyboardInterrupt, ValidatorVersionError): try: # tell the validator to not send any more messages self._unregister() while True: if fut is not None: # process futures as long as the tp has them, # if the TP_PROCESS_REQUEST doesn't come from # zeromq->asyncio in 1 second raise a # concurrent.futures.TimeOutError and be done. self._process_future(fut, 1, sigint=True) fut = self._stream.receive() except concurrent.futures.TimeoutError: # Where the tp will usually exit after # a KeyboardInterrupt. Caused by the 1 second # timeout in _process_future. pass except FutureTimeoutError: # If the validator is not able to respond to the # unregister request, exit. pass except RuntimeError as e: LOGGER.error("Error: %s", e) self.stop() def stop(self): """Closes the connection between the TransactionProcessor and the validator. """ self._stream.close()
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): self._stream = Stream(endpoint) startup_state = self._register() # Validators version 1.1 send startup info with the registration # response; newer versions will send an activation message with the # startup info if startup_state is None: startup_state = self._wait_until_active() self._updates = Queue() driver_thread = Thread( target=self._driver_loop) driver_thread.start() try: self._engine.start( self._updates, ZmqService( stream=self._stream, timeout=SERVICE_TIMEOUT), startup_state) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught engine exception") self.stop() driver_thread.join() def _driver_loop(self): try: future = self._stream.receive() while True: if self._exit: self._engine.stop() break try: message = future.result(1) future = self._stream.receive() except concurrent.futures.TimeoutError: continue result = self._process(message) self._updates.put(result) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught driver exception") def stop(self): self._exit = True self._engine.stop() self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ).SerializeToString() while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if ( response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY ): continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: if ( response.HasField('chain_head') and response.HasField('local_peer_info') ): return StartupState( response.chain_head, response.peers, response.local_peer_info) return None raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _wait_until_active(self): future = self._stream.receive() while True: try: message = future.result(1) except concurrent.futures.TimeoutError: continue if ( message.message_type == Message.CONSENSUS_NOTIFY_ENGINE_ACTIVATED ): notification = \ consensus_pb2.ConsensusNotifyEngineActivated() notification.ParseFromString(message.content) startup_state = StartupState( notification.chain_head, notification.peers, notification.local_peer_info) LOGGER.info( 'Received activation message with startup state: %s', startup_state) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck() .SerializeToString()) return startup_state LOGGER.warning('Received message type %s while waiting for \ activation message', message.message_type) future = self._stream.receive() def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) header = consensus_pb2.ConsensusPeerMessageHeader() header.ParseFromString(notification.message.header) peer_message = PeerMessage( header=header, header_bytes=notification.message.header, header_signature=notification.message.header_signature, content=notification.message.content) data = peer_message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_ENGINE_DEACTIVATED: self.stop() data = None else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
class ZmqDriver(Driver): def __init__(self, engine): super().__init__(engine) self._engine = engine self._stream = None self._exit = False self._updates = None def start(self, endpoint): self._stream = Stream(endpoint) startup_state = self._register() self._updates = Queue() driver_thread = Thread( target=self._driver_loop) driver_thread.start() try: self._engine.start( self._updates, ZmqService( stream=self._stream, timeout=SERVICE_TIMEOUT, name=self._engine.name(), version=self._engine.version()), startup_state) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught engine exception") self.stop() driver_thread.join() def _driver_loop(self): try: future = self._stream.receive() while True: if self._exit: self._engine.stop() break try: message = future.result(1) future = self._stream.receive() except concurrent.futures.TimeoutError: continue result = self._process(message) self._updates.put(result) except Exception: # pylint: disable=broad-except LOGGER.exception("Uncaught driver exception") def stop(self): self._exit = True self._engine.stop() self._stream.close() def _register(self): self._stream.wait_for_ready() request = consensus_pb2.ConsensusRegisterRequest( name=self._engine.name(), version=self._engine.version(), ).SerializeToString() while True: future = self._stream.send( message_type=Message.CONSENSUS_REGISTER_REQUEST, content=request) response = consensus_pb2.ConsensusRegisterResponse() response.ParseFromString(future.result(REGISTER_TIMEOUT).content) if ( response.status == consensus_pb2.ConsensusRegisterResponse.NOT_READY ): continue if response.status == consensus_pb2.ConsensusRegisterResponse.OK: return StartupState( response.chain_head, response.peers, response.local_peer_info) raise exceptions.ReceiveError( 'Registration failed with status {}'.format(response.status)) def _process(self, message): type_tag = message.message_type if type_tag == Message.CONSENSUS_NOTIFY_PEER_CONNECTED: notification = consensus_pb2.ConsensusNotifyPeerConnected() notification.ParseFromString(message.content) data = notification.peer_info elif type_tag == Message.CONSENSUS_NOTIFY_PEER_DISCONNECTED: notification = consensus_pb2.ConsensusNotifyPeerDisconnected() notification.ParseFromString(message.content) data = notification.peer_id elif type_tag == Message.CONSENSUS_NOTIFY_PEER_MESSAGE: notification = consensus_pb2.ConsensusNotifyPeerMessage() notification.ParseFromString(message.content) data = notification.message, notification.sender_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_NEW: notification = consensus_pb2.ConsensusNotifyBlockNew() notification.ParseFromString(message.content) data = notification.block elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_VALID: notification = consensus_pb2.ConsensusNotifyBlockValid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_INVALID: notification = consensus_pb2.ConsensusNotifyBlockInvalid() notification.ParseFromString(message.content) data = notification.block_id elif type_tag == Message.CONSENSUS_NOTIFY_BLOCK_COMMIT: notification = consensus_pb2.ConsensusNotifyBlockCommit() notification.ParseFromString(message.content) data = notification.block_id else: raise exceptions.ReceiveError( 'Received unexpected message type: {}'.format(type_tag)) self._stream.send_back( message_type=Message.CONSENSUS_NOTIFY_ACK, correlation_id=message.correlation_id, content=consensus_pb2.ConsensusNotifyAck().SerializeToString()) return type_tag, data
class TransactionProcessor(object): def __init__(self, url): self._stream = Stream(url) self._url = url self._handlers = [] @property def zmq_id(self): return self._stream.zmq_id def add_handler(self, handler): """Add a transaction family handler :param handler: """ self._handlers.append(handler) def _matches(self, handler, header): return header.family_name == handler.family_name \ and header.family_version in handler.family_versions \ and header.payload_encoding in handler.encodings def _find_handler(self, header): """Find a handler for a particular (family_name, family_versions, payload_encoding) :param header transaction_pb2.TransactionHeader: :return: handler """ try: return next(handler for handler in self._handlers if self._matches(handler, header)) except StopIteration: LOGGER.debug("Missing handler for header: %s", header) return None def _register_requests(self): """Returns all of the TpRegisterRequests for handlers :return (list): list of TpRegisterRequests """ return itertools.chain.from_iterable( # flattens the nested list [[ TpRegisterRequest(family=n, version=v, encoding=e, namespaces=h.namespaces) for n, v, e in itertools.product( [h.family_name], h.family_versions, h.encodings) ] for h in self._handlers]) def _unregister_request(self): """Returns a single TP_UnregisterRequest that requests that the validator stop sending transactions for previously registered handlers. :return (processor_pb2.TpUnregisterRequest): """ return TpUnregisterRequest() def _process(self, msg): if msg.message_type != Message.TP_PROCESS_REQUEST: LOGGER.debug( "Transaction Processor recieved invalid message type. " "Message type should be TP_PROCESS_REQUEST," " but is %s", Message.MessageType.Name(msg.message_type)) return request = TpProcessRequest() request.ParseFromString(msg.content) state = State(self._stream, request.context_id) header = TransactionHeader() header.ParseFromString(request.header) try: if not self._stream.is_ready(): raise ValidatorConnectionError() handler = self._find_handler(header) if handler is None: return handler.apply(request, state) self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.OK).SerializeToString()) except InvalidTransaction as it: LOGGER.warning("Invalid Transaction %s", it) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INVALID_TRANSACTION, message=str(it), extended_data=it.extended_data).SerializeToString()) except ValidatorConnectionError as vce: # TP_PROCESS_REQUEST has made it through the # handler.apply and an INVALID_TRANSACTION would have been # sent back but the validator has disconnected and so it # doesn't care about the response. LOGGER.warning("during invalid transaction response: %s", vce) except InternalError as ie: LOGGER.warning("internal error: %s", ie) try: self._stream.send_back( message_type=Message.TP_PROCESS_RESPONSE, correlation_id=msg.correlation_id, content=TpProcessResponse( status=TpProcessResponse.INTERNAL_ERROR, message=str(ie), extended_data=ie.extended_data).SerializeToString()) except ValidatorConnectionError as vce: # Same as the prior except block, but an internal error has # happened, but because of the disconnect the validator # probably doesn't care about the response. LOGGER.warning("during internal error response: %s", vce) except ValidatorConnectionError as vce: # Somewhere within handler.apply a future resolved with an # error status that the validator has disconnected. There is # nothing left to do but reconnect. LOGGER.warning( "during handler.apply a future was resolved " "with error status: %s", vce) def _process_future(self, future, timeout=None, sigint=False): try: msg = future.result(timeout) except CancelledError: # This error is raised when Task.cancel is called on # disconnect from the validator in stream.py, for # this future. return if msg is RECONNECT_EVENT: if sigint is False: LOGGER.info("reregistering with validator") self._stream.wait_for_ready() self._register() else: LOGGER.debug('received message of type: %s', Message.MessageType.Name(msg.message_type)) if msg.message_type == Message.TP_PING: self._stream.send_back( message_type=Message.TP_PING_RESPONSE, correlation_id=msg.correlation_id, content=TpPingResponse( status=TpPingResponse.OK).SerializeToString()) return self._process(msg) def _register(self): futures = [] for message in self._register_requests(): self._stream.wait_for_ready() future = self._stream.send( message_type=Message.TP_REGISTER_REQUEST, content=message.SerializeToString()) futures.append(future) for future in futures: resp = TpRegisterResponse() try: resp.ParseFromString(future.result().content) LOGGER.info("register attempt: %s", TpRegisterResponse.Status.Name(resp.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on registration: %s", vce) def _unregister(self): message = self._unregister_request() self._stream.wait_for_ready() future = self._stream.send(message_type=Message.TP_UNREGISTER_REQUEST, content=message.SerializeToString()) response = TpUnregisterResponse() try: response.ParseFromString(future.result(1).content) LOGGER.info("unregister attempt: %s", TpUnregisterResponse.Status.Name(response.status)) except ValidatorConnectionError as vce: LOGGER.info("during waiting for response on unregistration: %s", vce) def start(self): fut = None try: self._register() while True: # During long running processing this # is where the transaction processor will # spend most of its time fut = self._stream.receive() self._process_future(fut) except KeyboardInterrupt: try: # tell the validator to not send any more messages self._unregister() while True: if fut is not None: # process futures as long as the tp has them, # if the TP_PROCESS_REQUEST doesn't come from # zeromq->asyncio in 1 second raise a # concurrent.futures.TimeOutError and be done. self._process_future(fut, 1, sigint=True) fut = self._stream.receive() except concurrent.futures.TimeoutError: # Where the tp will usually exit after # a KeyboardInterrupt. Caused by the 1 second # timeout in _process_future. pass except FutureTimeoutError: # If the validator is not able to respond to the # unregister request, exit. pass def stop(self): self._stream.close()
class Subscriber(object): """Creates an object that can subscribe to state delta events using the Sawtooth SDK's Stream class. Handler functions can be added prior to subscribing, and each will be called on each delta event received. """ def __init__(self, validator_url): LOGGER.info('Connecting to validator: %s', validator_url) self._stream = Stream(validator_url) self._event_handlers = [] self._is_active = False def add_handler(self, handler): """Adds a handler which will be passed state delta events when they occur. Note that this event is mutable. """ self._event_handlers.append(handler) def clear_handlers(self): """Clears any delta handlers. """ self._event_handlers = [] def start(self, known_ids=None): """Subscribes to state delta events, and then waits to receive deltas. Sends any events received to delta handlers. """ if not known_ids: known_ids = [NULL_BLOCK_ID] self._stream.wait_for_ready() LOGGER.debug('Subscribing to state delta events') block_sub = EventSubscription(event_type='sawtooth/block-commit') delta_sub = EventSubscription( event_type='sawtooth/state-delta', filters=[EventFilter( key='address', match_string='^{}.*'.format(NAMESPACE), filter_type=EventFilter.REGEX_ANY)]) request = ClientEventsSubscribeRequest( last_known_block_ids=known_ids, subscriptions=[block_sub, delta_sub]) response_future = self._stream.send( Message.CLIENT_EVENTS_SUBSCRIBE_REQUEST, request.SerializeToString()) response = ClientEventsSubscribeResponse() response.ParseFromString(response_future.result().content) # Forked all the way back to genesis, restart with no known_ids if (response.status == ClientEventsSubscribeResponse.UNKNOWN_BLOCK and known_ids): self.start() if response.status != ClientEventsSubscribeResponse.OK: raise RuntimeError( 'Subscription failed with status: {}'.format( ClientEventsSubscribeResponse.Status.Name( response.status))) self._is_active = True LOGGER.debug('Successfully subscribed to state delta events') while self._is_active: message_future = self._stream.receive() event_list = EventList() event_list.ParseFromString(message_future.result().content) for handler in self._event_handlers: handler(event_list.events) def stop(self): """Stops the Subscriber, unsubscribing from state delta events and closing the the stream's connection. """ self._is_active = False LOGGER.debug('Unsubscribing from state delta events') request = ClientEventsUnsubscribeRequest() response_future = self._stream.send( Message.CLIENT_EVENTS_UNSUBSCRIBE_REQUEST, request.SerializeToString()) response = ClientEventsUnsubscribeResponse() response.ParseFromString(response_future.result().content) if response.status != ClientEventsUnsubscribeResponse.OK: LOGGER.warning( 'Failed to unsubscribe with status: %s', ClientEventsUnsubscribeResponse.Status.Name(response.status)) self._stream.close()