def _sequential_add_batch_after_all_results_set(self, scheduler, context_manager): """Tests that adding a new batch only after setting all of the txn results will produce only expected state roots. Here no state roots were specified, so similar to block publishing use of scheduler. Basically: 1) Create 3 batches, the last being marked as having an invalid transaction. Add one batch and then while the scheduler keeps on returning transactions, set the txn result, and then call next_transaction. 2) Call finalize, and then assert that the scheduler is complete 3) Assert that the first batch is valid and has no state hash, the second batch is valid and since it is the last valid batch in the scheduler has a state hash, and that the third batch is invalid and consequently has no state hash. """ private_key = self._context.new_random_private_key() signer = self._crypto_factory.new_signer(private_key) # 1) batch_signatures = [] batches = [] for names in [['a', 'b'], ['d', 'e'], ['invalid', 'c']]: batch_txns = [] for name in names: txn, _ = create_transaction( payload=name.encode(), signer=signer) batch_txns.append(txn) batch = create_batch( transactions=batch_txns, signer=signer) batches.append(batch) batch_signatures.append(batch.header_signature) invalid_payload_sha = hashlib.sha512( 'invalid'.encode()).hexdigest() for batch in batches: scheduler.add_batch(batch=batch) txn_info = scheduler.next_transaction() while txn_info is not None: txn_header = transaction_pb2.TransactionHeader() txn_header.ParseFromString(txn_info.txn.header) inputs_outputs = list(txn_header.inputs) c_id = context_manager.create_context( state_hash=context_manager.get_first_root(), base_contexts=txn_info.base_context_ids, inputs=list(txn_header.inputs), outputs=list(txn_header.outputs)) context_manager.set( context_id=c_id, address_value_list=[{inputs_outputs[0]: b'5'}]) if txn_header.payload_sha512 == invalid_payload_sha: scheduler.set_transaction_execution_result( txn_info.txn.header_signature, is_valid=False, context_id=None) else: scheduler.set_transaction_execution_result( txn_info.txn.header_signature, is_valid=True, context_id=c_id) txn_info = scheduler.next_transaction() # 2) scheduler.finalize() self.assertTrue(scheduler.complete(block=False), "The scheduler has had all txn results set so after " " calling finalize the scheduler is complete") # 3) first_batch_id = batch_signatures.pop(0) result1 = scheduler.get_batch_execution_result(first_batch_id) self.assertEqual( result1.is_valid, True, "The first batch is valid") self.assertIsNone(result1.state_hash, "The first batch doesn't produce" " a state hash") second_batch_id = batch_signatures.pop(0) result2 = scheduler.get_batch_execution_result(second_batch_id) self.assertEqual( result2.is_valid, True, "The second batch is valid") self.assertIsNotNone(result2.state_hash, "The second batch is the " "last valid batch in the " "scheduler") third_batch_id = batch_signatures.pop(0) result3 = scheduler.get_batch_execution_result(third_batch_id) self.assertEqual(result3.is_valid, False) self.assertIsNone(result3.state_hash, "The last batch is invalid so " "doesn't have a state hash")
def do_genesis(args): """Executes the `poet genesis` subcommand. This command generates a validator registry transaction and saves it to a file, whose location is determined by the args. The signup data, generated by the selected enclave, is also stored in a well-known location. """ public_key, signing_key = _read_signing_keys(args.key) public_key_hash = sha256(public_key.encode()).hexdigest() with PoetEnclaveModuleWrapper( enclave_module=args.enclave_module, config_dir=config.get_config_dir(), data_dir=config.get_data_dir()) as poet_enclave_module: signup_info = SignupInfo.create_signup_info( poet_enclave_module=poet_enclave_module, originator_public_key_hash=public_key_hash, nonce=SignupInfo.block_id_to_nonce(NULL_BLOCK_IDENTIFIER)) print( 'Writing key state for PoET public key: {}...{}'.format( signup_info.poet_public_key[:8], signup_info.poet_public_key[-8:])) # Store the newly-created PoET key state, associating it with its # corresponding public key poet_key_state_store = \ PoetKeyStateStore( data_dir=config.get_data_dir(), validator_id=public_key) poet_key_state_store[signup_info.poet_public_key] = \ PoetKeyState( sealed_signup_data=signup_info.sealed_signup_data, has_been_refreshed=False) # Create the validator registry payload payload = \ vr_pb.ValidatorRegistryPayload( verb='register', name='validator-{}'.format(public_key[:8]), id=public_key, signup_info=vr_pb.SignUpInfo( poet_public_key=signup_info.poet_public_key, proof_data=signup_info.proof_data, anti_sybil_id=signup_info.anti_sybil_id, nonce=NULL_BLOCK_IDENTIFIER)) serialized = payload.SerializeToString() # Create the address that will be used to look up this validator # registry transaction. Seems like a potential for refactoring.. validator_entry_address = \ VR_NAMESPACE + sha256(public_key.encode()).hexdigest() # Create a transaction header and transaction for the validator # registry update amd then hand it off to the batch publisher to # send out. output_addresses = [validator_entry_address, VALIDATOR_MAP_ADDRESS] input_addresses = \ output_addresses + \ [SettingsView.setting_address('sawtooth.poet.report_public_key_pem'), SettingsView.setting_address('sawtooth.poet.' 'valid_enclave_measurements'), SettingsView.setting_address('sawtooth.poet.valid_enclave_basenames')] header = \ txn_pb.TransactionHeader( signer_public_key=public_key, family_name='sawtooth_validator_registry', family_version='1.0', inputs=input_addresses, outputs=output_addresses, dependencies=[], payload_sha512=sha512(serialized).hexdigest(), batcher_public_key=public_key, nonce=time.time().hex().encode()).SerializeToString() signature = signing.sign(header, signing_key) transaction = \ txn_pb.Transaction( header=header, payload=serialized, header_signature=signature) batch = _create_batch(public_key, signing_key, [transaction]) batch_list = batch_pb.BatchList(batches=[batch]) try: print('Generating {}'.format(args.output)) with open(args.output, 'wb') as batch_file: batch_file.write(batch_list.SerializeToString()) except IOError as e: raise CliException( 'Unable to write to batch file: {}'.format(str(e)))
def _add_valid_batch_invalid_batch(self, scheduler, context_manager): """Tests the squash function. That the correct state hash is found at the end of valid and invalid batches, similar to block publishing. Basically: 1. Adds two batches, one where all the txns are valid, and one where one of the txns is invalid. 2. Run through the scheduler executor interaction as txns are processed. 3. Verify that the state root obtained through the squash function is the same as directly updating the merkle tree. 4. Verify that correct batch statuses are set This test should work for both a serial and parallel scheduler. """ private_key = self._context.new_random_private_key() signer = self._crypto_factory.new_signer(private_key) # 1) batch_signatures = [] for names in [['a', 'b'], ['invalid', 'c'], ['d', 'e']]: batch_txns = [] for name in names: txn, _ = create_transaction( payload=name.encode(), signer=signer) batch_txns.append(txn) batch = create_batch( transactions=batch_txns, signer=signer) batch_signatures.append(batch.header_signature) scheduler.add_batch(batch) scheduler.finalize() # 2) sched1 = iter(scheduler) invalid_payload = hashlib.sha512('invalid'.encode()).hexdigest() while not scheduler.complete(block=False): try: txn_info = next(sched1) except StopIteration: break txn_header = transaction_pb2.TransactionHeader() txn_header.ParseFromString(txn_info.txn.header) inputs_or_outputs = list(txn_header.inputs) c_id = context_manager.create_context( state_hash=txn_info.state_hash, inputs=inputs_or_outputs, outputs=inputs_or_outputs, base_contexts=txn_info.base_context_ids) if txn_header.payload_sha512 == invalid_payload: scheduler.set_transaction_execution_result( txn_info.txn.header_signature, False, None) else: context_manager.set(c_id, [{inputs_or_outputs[0]: b"1"}]) scheduler.set_transaction_execution_result( txn_info.txn.header_signature, True, c_id) sched2 = iter(scheduler) # 3) txn_info_a = next(sched2) address_a = _get_address_from_txn(txn_info_a) txn_info_b = next(sched2) address_b = _get_address_from_txn(txn_info_b) next(sched2) txn_info_d = next(sched2) address_d = _get_address_from_txn(txn_info_d) txn_info_e = next(sched2) address_e = _get_address_from_txn(txn_info_e) merkle_database = MerkleDatabase(dict_database.DictDatabase()) state_root_end = merkle_database.update( {address_a: b"1", address_b: b"1", address_d: b"1", address_e: b"1"}, virtual=False) # 4) batch1_result = scheduler.get_batch_execution_result( batch_signatures[0]) self.assertTrue(batch1_result.is_valid) batch2_result = scheduler.get_batch_execution_result( batch_signatures[1]) self.assertFalse(batch2_result.is_valid) batch3_result = scheduler.get_batch_execution_result( batch_signatures[2]) self.assertTrue(batch3_result.is_valid) self.assertEqual(batch3_result.state_hash, state_root_end)
def _get_address_from_txn(txn_info): txn_header = transaction_pb2.TransactionHeader() txn_header.ParseFromString(txn_info.txn.header) inputs_or_outputs = list(txn_header.inputs) address_b = inputs_or_outputs[0] return address_b
def _register_signup_information(self, block_header, poet_enclave_module): # Create signup information for this validator, putting the block ID # of the block previous to the block referenced by block_header in the # nonce. Block ID is better than wait certificate ID for testing # freshness as we need to account for non-PoET blocks. public_key_hash = \ hashlib.sha256( block_header.signer_pubkey.encode()).hexdigest() signup_info = \ SignupInfo.create_signup_info( poet_enclave_module=poet_enclave_module, validator_address=block_header.signer_pubkey, originator_public_key_hash=public_key_hash, nonce=block_header.previous_block_id) # Create the validator registry payload payload = \ vr_pb.ValidatorRegistryPayload( verb='register', name='validator-{}'.format(block_header.signer_pubkey[:8]), id=block_header.signer_pubkey, signup_info=vr_pb.SignUpInfo( poet_public_key=signup_info.poet_public_key, proof_data=signup_info.proof_data, anti_sybil_id=signup_info.anti_sybil_id, nonce=block_header.previous_block_id), ) serialized = payload.SerializeToString() # Create the address that will be used to look up this validator # registry transaction. Seems like a potential for refactoring.. validator_entry_address = \ PoetBlockPublisher._validator_registry_namespace + \ hashlib.sha256(block_header.signer_pubkey.encode()).hexdigest() # Create a transaction header and transaction for the validator # registry update amd then hand it off to the batch publisher to # send out. addresses = \ [validator_entry_address, PoetBlockPublisher._validator_map_address] header = \ txn_pb.TransactionHeader( signer_pubkey=block_header.signer_pubkey, family_name='sawtooth_validator_registry', family_version='1.0', inputs=addresses, outputs=addresses, dependencies=[], payload_encoding="application/protobuf", payload_sha512=hashlib.sha512(serialized).hexdigest(), batcher_pubkey=block_header.signer_pubkey, nonce=time.time().hex().encode()).SerializeToString() signature = \ signing.sign(header, self._batch_publisher.identity_signing_key) transaction = \ txn_pb.Transaction( header=header, payload=serialized, header_signature=signature) LOGGER.info( 'Register Validator Name=%s, ID=%s...%s, PoET public key=%s...%s, ' 'Nonce=%s', payload.name, payload.id[:8], payload.id[-8:], payload.signup_info.poet_public_key[:8], payload.signup_info.poet_public_key[-8:], block_header.previous_block_id[:8]) self._batch_publisher.send([transaction]) # Store the key state so that we can look it up later if need be LOGGER.info('Save key state PPK=%s...%s => SSD=%s...%s', signup_info.poet_public_key[:8], signup_info.poet_public_key[-8:], signup_info.sealed_signup_data[:8], signup_info.sealed_signup_data[-8:]) self._poet_key_state_store[signup_info.poet_public_key] = \ PoetKeyState( sealed_signup_data=signup_info.sealed_signup_data, has_been_refreshed=False) # Cache the PoET public key in a class to indicate that this is the # current public key for the PoET enclave PoetBlockPublisher._poet_public_key = signup_info.poet_public_key
def _future_done_callback(self, request, result): """ :param request (bytes):the serialized request :param result (FutureResult): """ self._in_process_transactions_count.dec() req = processor_pb2.TpProcessRequest() req.ParseFromString(request) # If raw header bytes were sent then deserialize to get # transaction family name and version if req.header_bytes == b'': # When header_bytes field is empty, the header field will # be populated, so we can use that directly. request_header = req.header else: # Deserialize the header_bytes request_header = transaction_pb2.TransactionHeader() request_header.ParseFromString(req.header_bytes) response = processor_pb2.TpProcessResponse() response.ParseFromString(result.content) processor_type = ProcessorType(request_header.family_name, request_header.family_version) self._processor_manager[processor_type].get_processor( result.connection_id).dec_occupancy() self._processor_manager.notify() self._get_tp_process_response_counter( response.Status.Name(response.status)).inc() if result.connection_id in self._open_futures and \ req.signature in self._open_futures[result.connection_id]: del self._open_futures[result.connection_id][req.signature] if response.status == processor_pb2.TpProcessResponse.OK: state_sets, state_deletes, events, data = \ self._context_manager.get_execution_results(req.context_id) state_changes = [ transaction_receipt_pb2.StateChange( address=addr, value=value, type=transaction_receipt_pb2.StateChange.SET) for addr, value in state_sets.items() ] + [ transaction_receipt_pb2.StateChange( address=addr, type=transaction_receipt_pb2.StateChange.DELETE) for addr in state_deletes ] self._scheduler.set_transaction_execution_result( txn_signature=req.signature, is_valid=True, context_id=req.context_id, state_changes=state_changes, events=events, data=data) elif response.status == processor_pb2.TpProcessResponse.INTERNAL_ERROR: LOGGER.error( "Transaction processor internal error: %s " "(transaction: %s, name: %s, version: %s)", response.message, req.signature, request_header.family_name, request_header.family_version) # Make sure that the transaction wasn't unscheduled in the interim if self._scheduler.is_transaction_in_schedule(req.signature): self._execute(processor_type=processor_type, process_request=req) else: self._context_manager.delete_contexts( context_id_list=[req.context_id]) self._fail_transaction(txn_signature=req.signature, context_id=req.context_id, error_message=response.message, error_data=response.extended_data)
def _execute_schedule(self): for txn_info in self._scheduler: self._transaction_execution_count.inc() txn = txn_info.txn header = transaction_pb2.TransactionHeader() header.ParseFromString(txn.header) processor_type = ProcessorType(header.family_name, header.family_version) config = self._settings_view_factory.create_settings_view( txn_info.state_hash) transaction_families = config.get_setting( key=self._tp_settings_key, default_value="[]") # After reading the transaction families required in configuration # try to json.loads them into a python object # If there is a misconfiguration, proceed as if there is no # configuration. try: transaction_families = json.loads(transaction_families) required_transaction_processors = [ ProcessorType(d.get('family'), d.get('version')) for d in transaction_families ] except ValueError: LOGGER.error( "sawtooth.validator.transaction_families " "misconfigured. Expecting a json array, found" " %s", transaction_families) required_transaction_processors = [] # First check if the transaction should be failed # based on configuration if required_transaction_processors and \ processor_type not in required_transaction_processors: # The txn processor type is not in the required # transaction processors so # failing transaction right away LOGGER.debug( "failing transaction %s of type (name=%s," "version=%s) since it isn't" " required in the configuration", txn.header_signature, processor_type.name, processor_type.version) self._fail_transaction(txn.header_signature) continue if processor_type in required_transaction_processors: # The txn processor type is in the required # transaction processors: check all the outputs of # the transaction match one namespace listed transaction_family = \ next(t for t in transaction_families if t.get('family') == header.family_name and t.get('version') == header.family_version) # if no namespaces are indicated, then the empty prefix is # inserted by default namespaces = transaction_family.get('namespaces', ['']) if not isinstance(namespaces, list): LOGGER.error( "namespaces should be a list for " "transaction family (name=%s, version=%s)", processor_type.name, processor_type.version) prefixes = header.outputs bad_prefixes = [ prefix for prefix in prefixes if not any(prefix.startswith(n) for n in namespaces) ] for prefix in bad_prefixes: # log each LOGGER.debug( "failing transaction %s of type (name=%s," "version=%s) because of no namespace listed " "in %s from the configuration settings can " "match the prefix %s", txn.header_signature, processor_type.name, processor_type.version, namespaces, prefix) if bad_prefixes: self._fail_transaction(txn.header_signature) continue try: context_id = self._context_manager.create_context( state_hash=txn_info.state_hash, base_contexts=txn_info.base_context_ids, inputs=list(header.inputs), outputs=list(header.outputs)) except KeyError: LOGGER.error( "Error creating context for transaction %s, " "scheduler provided a base context that was not " "in the context manager.", txn.header_signature) self._scheduler.set_transaction_execution_result( txn_signature=txn.header_signature, is_valid=False, context_id=None) continue except CreateContextException: LOGGER.exception("Exception creating context") self._scheduler.set_transaction_execution_result( txn_signature=txn.header_signature, is_valid=False, context_id=None) continue process_request = processor_pb2.TpProcessRequest( header=header, payload=txn.payload, signature=txn.header_signature, context_id=context_id, header_bytes=txn.header) # Since we have already checked if the transaction should be failed # all other cases should either be executed or waited for. self._execute(processor_type=processor_type, process_request=process_request) self._done = True