def _create_transactions(self, count): txn_list = [] for i in range(count): payload = {'Verb': 'set', 'Name': 'name', 'Value': 1} intkey_prefix = \ hashlib.sha512('intkey'.encode('utf-8')).hexdigest()[0:6] addr = intkey_prefix + \ hashlib.sha512(payload["Name"].encode('utf-8')).hexdigest() payload_encode = hashlib.sha512(cbor.dumps(payload)).hexdigest() header = TransactionHeader(signer_pubkey=self.public_key, family_name='intkey', family_version='1.0', inputs=[addr], outputs=[addr], dependencies=[], payload_sha512=payload_encode) header.batcher_pubkey = self.public_key header_bytes = header.SerializeToString() signature = signing.sign(header_bytes, self.private_key) transaction = Transaction(header=header_bytes, payload=cbor.dumps(payload), header_signature=signature) txn_list.append(transaction) return txn_list
def _do_local(self, transactions, expected_signer, arguments): """ A transaction must be signed by the same key as the block. This rule takes a list of transaction indices in the block and enforces the rule on each. This rule is useful in combination with the other rules to ensure a client is not submitting transactions that should only be injected by the winning validator. """ indices = arguments.split(",") txns_len = len(transactions) for index in indices: try: index = int(index.strip()) except ValueError: LOGGER.warning( "Ignore, local requries one or more comma " "seperated integers that represent indices, not" " %s", arguments) return True if abs(index) >= txns_len: LOGGER.debug( "Ignore, Block does not have enough " "transactions to validate this rule local:%s", index) continue txn = transactions[index] header = TransactionHeader() header.ParseFromString(txn.header) if header.signer_public_key != expected_signer: LOGGER.debug( "Transaction at postion %s was not signed by the" " same key as the block.", index) return False return True
def _do_nofx(self, transactions, arguments): """ Only N of transaction type X may be included in a block. The first argument must be interpretable as an integer. The second argument is interpreted as the name of a transaction family. For example, the string "NofX:2,intkey" means only allow 2 intkey transactions per block. """ try: num, family = arguments.split(',') limit = int(num.strip()) except ValueError: LOGGER.warning( "Ignore, NofX requires arguments in the format " "int,family not %s", arguments) return True count = 0 family = family.strip() for txn in transactions: header = TransactionHeader() header.ParseFromString(txn.header) if header.family_name == family: count += 1 if count > limit: LOGGER.debug("Too many transactions of type %s", family) return False return True
def create_transaction(signer, verb, name, value): payload = cbor.dumps({ 'Verb': verb, 'Name': name, 'Value': value }, sort_keys=True) addresses = [make_intkey_address(name)] nonce = hex(random.randint(0, 2**64)) txn_pub_key = signer.get_public_key().as_hex() header = TransactionHeader( signer_public_key=txn_pub_key, family_name="intkey", family_version="1.0", inputs=addresses, outputs=addresses, dependencies=[], payload_sha512=hashlib.sha512(payload).hexdigest(), batcher_public_key=signer.get_public_key().as_hex(), nonce=nonce) signature = signer.sign(header.SerializeToString()) return Transaction(header=header.SerializeToString(), payload=payload, header_signature=signature)
def validate_transaction(self, txn): # validate transactions signature header = TransactionHeader() header.ParseFromString(txn.header) valid = signing.verify(txn.header, txn.header_signature, header.signer_pubkey) return valid
def _complete_batch(self, batch): valid = True dependencies = [] for txn in batch.transactions: txn_header = TransactionHeader() txn_header.ParseFromString(txn.header) for dependency in txn_header.dependencies: # Check to see if the dependency has been seen or is committed if dependency not in self._seen_txns and not \ self._transaction_committed(dependency): self._unsatisfied_dependency_count.inc() # Check to see if the dependency has already been requested if dependency not in self._requested: dependencies.append(dependency) self._requested[dependency] = None if dependency not in self._incomplete_batches: self._incomplete_batches[dependency] = [batch] elif batch not in self._incomplete_batches[dependency]: self._incomplete_batches[dependency] += [batch] valid = False if not valid: self._gossip.broadcast_batch_by_transaction_id_request( dependencies) return valid
def _dependency_not_processed(self, txn): header = TransactionHeader() header.ParseFromString(txn.header) if any(not self._all_in_batch_have_results(d) for d in list(header.dependencies)): return True return False
def _generate_batch(self, payload): payload_encoded = payload.encode('utf-8') hasher = hashlib.sha512() hasher.update(payload_encoded) header = TransactionHeader() header.batcher_pubkey = self.public_key # txn.dependencies not yet header.family_name = 'test' header.family_version = '1' header.nonce = _generate_id(16) header.payload_encoding = "text" header.payload_sha512 = hasher.hexdigest().encode() header.signer_pubkey = self.public_key txn = Transaction() header_bytes = header.SerializeToString() txn.header = header_bytes txn.header_signature = signing.sign(header_bytes, self.signing_key) txn.payload = payload_encoded batch_header = BatchHeader() batch_header.signer_pubkey = self.public_key batch_header.transaction_ids.extend([txn.header_signature]) batch = Batch() header_bytes = batch_header.SerializeToString() batch.header = header_bytes batch.header_signature = signing.sign(header_bytes, self.signing_key) batch.transactions.extend([txn]) return batch
def _validate_transactions_in_batch(batch, chain_commit_state): """Verify that all transactions in this batch are unique and that all transaction dependencies in this batch have been satisfied. :param batch: the batch to verify :param chain_commit_state: the current chain commit state to verify the batch against :return: Boolean: True if all dependencies are present and all transactions are unique. """ for txn in batch.transactions: txn_hdr = TransactionHeader() txn_hdr.ParseFromString(txn.header) if chain_commit_state.has_transaction(txn.header_signature): LOGGER.debug( "Batch invalid due to duplicate transaction: %s", txn.header_signature[:8]) return False for dep in txn_hdr.dependencies: if not chain_commit_state.has_transaction(dep): LOGGER.debug( "Batch invalid due to missing transaction dependency;" " transaction %s depends on %s", txn.header_signature[:8], dep[:8]) return False return True
def is_valid_batch(batch): # validate batch signature header = BatchHeader() header.ParseFromString(batch.header) if not signing.verify(batch.header, batch.header_signature, header.signer_public_key): LOGGER.debug("batch failed signature validation: %s", batch.header_signature) return False # validate all transactions in batch for txn in batch.transactions: if not is_valid_transaction(txn): return False txn_header = TransactionHeader() txn_header.ParseFromString(txn.header) if txn_header.batcher_public_key != header.signer_public_key: LOGGER.debug("txn batcher public_key does not match signer" "public_key for batch: %s txn: %s", batch.header_signature, txn.header_signature) return False return True
def is_valid_batch(batch): # validate batch signature header = BatchHeader() header.ParseFromString(batch.header) context = create_context('secp256k1') public_key = Secp256k1PublicKey.from_hex(header.signer_public_key) if not context.verify(batch.header_signature, batch.header, public_key): LOGGER.debug("batch failed signature validation: %s", batch.header_signature) return False # validate all transactions in batch for txn in batch.transactions: if not is_valid_transaction(txn): return False txn_header = TransactionHeader() txn_header.ParseFromString(txn.header) if txn_header.batcher_public_key != header.signer_public_key: LOGGER.debug( "txn batcher public_key does not match signer" "public_key for batch: %s txn: %s", batch.header_signature, txn.header_signature) return False return True
def _complete_batch(self, batch): valid = True dependencies = [] for txn in batch.transactions: txn_header = TransactionHeader() txn_header.ParseFromString(txn.header) for dependency in txn_header.dependencies: # Check to see if the dependency has been seen or is in the # current chain (block_store) if dependency not in self._seen_txns and not \ self.block_cache.block_store.has_transaction( dependency): LOGGER.debug( "Transaction %s in batch %s has unsatisfied dependency: %s", txn.header_signature, batch.header_signature, dependency) # Check to see if the dependency has already been requested if dependency not in self._requested: dependencies.append(dependency) self._requested[dependency] = None if dependency not in self._incomplete_batches: self._incomplete_batches[dependency] = [batch] elif batch not in self._incomplete_batches[dependency]: self._incomplete_batches[dependency] += [batch] valid = False if not valid: self.gossip.broadcast_batch_by_transaction_id_request(dependencies) return valid
def check_for_transaction_dependencies(self, transactions): """Check that all explicit dependencies in all transactions passed have been satisfied.""" dependencies = [] txn_ids = [] for txn in transactions: txn_ids.append(txn.header_signature) txn_hdr = TransactionHeader() txn_hdr.ParseFromString(txn.header) dependencies.extend(txn_hdr.dependencies) for dep in dependencies: # Check for dependency within the given block's batches if dep in txn_ids: continue # Check for dependency in the uncommitted blocks if dep in self.uncommitted_txn_ids: continue # Check for dependency in the committe blocks if self.block_store.has_transaction(dep): committed_block =\ self.block_store.get_block_by_transaction_id(dep) # Make sure the block wouldn't be uncomitted if the given block # were uncommitted if self._block_in_chain(committed_block): continue raise MissingDependency(dep)
def next_transaction(self): with self._condition: # We return the next transaction which hasn't been scheduled and # is not blocked by a dependency. next_txn = None no_longer_available = [] for txn_id, txn in self._txns_available.items(): if (self._has_predecessors(txn_id) or self._is_outstanding(txn_id)): continue header = TransactionHeader() header.ParseFromString(txn.header) deps = tuple(header.dependencies) if self._dependency_not_processed(deps): continue if self._txn_failed_by_dep(deps): no_longer_available.append(txn_id) self._txn_results[txn_id] = \ TxnExecutionResult( signature=txn_id, is_valid=False, context_id=None, state_hash=None) continue if not self._txn_is_in_valid_batch(txn_id) and \ self._can_fail_fast(txn_id): self._txn_results[txn_id] = \ TxnExecutionResult( signature=txn_id, is_valid=False, context_id=None, state_hash=None) no_longer_available.append(txn_id) continue next_txn = txn break for txn_id in no_longer_available: del self._txns_available[txn_id] if next_txn is not None: bases = self._get_initial_state_for_transaction(next_txn) info = TxnInformation( txn=next_txn, state_hash=self._first_state_hash, base_context_ids=bases) self._scheduled.append(next_txn.header_signature) del self._txns_available[next_txn.header_signature] self._scheduled_txn_info[next_txn.header_signature] = info return info return None
def create_intkey_transaction(verb, name, value, deps, signer): payload = IntKeyPayload(verb=verb, name=name, value=value) # The prefix should eventually be looked up from the # validator's namespace registry. addr = make_intkey_address(name) header = TransactionHeader( signer_public_key=signer.get_public_key().as_hex(), family_name='intkey', family_version='1.0', inputs=[addr], outputs=[addr], dependencies=deps, payload_sha512=payload.sha512(), batcher_public_key=signer.get_public_key().as_hex()) header_bytes = header.SerializeToString() signature = signer.sign(header_bytes) transaction = Transaction(header=header_bytes, payload=payload.to_cbor(), header_signature=signature) return transaction
def is_transaction_signer_authorized(self, transactions, identity_view): """ Check the transaction signing key against the allowed transactor permissions. The roles being checked are the following, from first to last: "transactor.transaction_signer.<TP_Name>" "transactor.transaction_signer" "transactor" "default" The first role that is set will be the one used to enforce if the transaction signer is allowed. Args: transactions (List of Transactions): The transactions that are being verified. identity_view (IdentityView): The IdentityView that should be used to verify the transactions. """ role = None if role is None: role = identity_view.get_role("transactor.transaction_signer") if role is None: role = identity_view.get_role("transactor") if role is None: policy_name = "default" else: policy_name = role.policy_name policy = identity_view.get_policy(policy_name) family_roles = {} for transaction in transactions: header = TransactionHeader() header.ParseFromString(transaction.header) family_policy = None if header.family_name not in family_roles: role = identity_view.get_role( "transactor.transaction_signer." + header.family_name) if role is not None: family_policy = identity_view.get_policy(role.policy_name) family_roles[header.family_name] = family_policy else: family_policy = family_roles[header.family_name] if family_policy is not None: if not self._allowed(header.signer_pubkey, family_policy): LOGGER.debug("Transaction Signer: %s is not permitted.", header.signer_pubkey) return False else: if policy is not None: if not self._allowed(header.signer_pubkey, policy): LOGGER.debug( "Transaction Signer: %s is not permitted.", header.signer_pubkey) return False return True
def validate_batch(batch): # validate batch signature header = BatchHeader() header.ParseFromString(batch.header) valid = signing.verify(batch.header, batch.header_signature, header.signer_pubkey) if not valid: LOGGER.debug("batch failed signature validation: %s", batch.header_signature) # validate all transactions in batch total = len(batch.transactions) index = 0 while valid and index < total: txn = batch.transactions[index] valid = validate_transaction(txn) if valid: txn_header = TransactionHeader() txn_header.ParseFromString(txn.header) if txn_header.batcher_pubkey != header.signer_pubkey: LOGGER.debug( "txn batcher pubkey does not match signer" "pubkey for batch: %s txn: %s", batch.header_signature, txn.header_signature) valid = False index += 1 return valid
def _txn_failed_by_dep(self, txn): header = TransactionHeader() header.ParseFromString(txn.header) if any( self._any_in_batch_are_invalid(d) for d in list(header.dependencies)): return True return False
def add_batch(self, batch, state_hash=None): with self._condition: if self._final: raise SchedulerError('Invalid attempt to add batch to ' 'finalized scheduler; batch: {}'.format( batch.header_signature)) if not self._batches: self._least_batch_id_wo_results = batch.header_signature self._batches.append(batch) self._batches_by_id[batch.header_signature] = batch for txn in batch.transactions: self._batches_by_txn_id[txn.header_signature] = batch self._txns_available.append(txn) self._transactions[txn.header_signature] = txn if state_hash is not None: b_id = batch.header_signature self._batches_with_state_hash[b_id] = state_hash # For dependency handling: First, we determine our dependencies # based on the current state of the predecessor tree. Second, # we update the predecessor tree with reader and writer # information based on input and outputs. for txn in batch.transactions: header = TransactionHeader() header.ParseFromString(txn.header) # Calculate predecessors (transaction ids which must come # prior to the current transaction). predecessors = self._find_input_dependencies(header.inputs) predecessors.extend( self._find_output_dependencies(header.outputs)) txn_id = txn.header_signature # Update our internal state with the computed predecessors. self._txn_predecessors[txn_id] = list(set(predecessors)) # Update the predecessor tree. # # Order of reader/writer operations is relevant. A writer # may overshadow a reader. For example, if the transaction # has the same input/output address, the end result will be # this writer (txn.header_signature) stored at the address of # the predecessor tree. The reader information will have been # discarded. Write operations to partial addresses will also # overshadow entire parts of the predecessor tree. # # Thus, the order here (inputs then outputs) will cause the # minimal amount of relevant information to be stored in the # predecessor tree, with duplicate information being # automatically discarded by the set_writer() call. for address in header.inputs: self._predecessor_tree.add_reader(address, txn_id) for address in header.outputs: self._predecessor_tree.set_writer(address, txn_id) self._condition.notify_all()
def _make_mock_transaction(self, txn_id='txn_id', payload='payload'): header = TransactionHeader(batcher_pubkey='pubkey', family_name='family', family_version='0.0', nonce=txn_id, signer_pubkey='pubkey') return Transaction(header=header.SerializeToString(), header_signature=txn_id, payload=payload.encode())
def _make_mock_transaction(base_id='id', payload='payload'): txn_id = 'c' * (128 - len(base_id)) + base_id header = TransactionHeader(batcher_public_key='public_key-' + base_id, family_name='family', family_version='0.0', nonce=txn_id, signer_public_key='public_key-' + base_id) return Transaction(header=header.SerializeToString(), header_signature=txn_id, payload=payload.encode())
def validate_transaction(self, txn): # validate transactions signature recovered_pubkey = signing.recover_pubkey(txn.header, txn.header_signature) header = TransactionHeader() header.ParseFromString(txn.header) if recovered_pubkey == header.signer_pubkey: return True else: return False
def _create_transactions(self, count, matched_payload=True, valid_signature=True, valid_batcher=True): txn_list = [] for i in range(count): payload = {'Verb': 'set', 'Name': 'name' + str(random.randint(0, 100)), 'Value': random.randint(0, 100)} intkey_prefix = \ hashlib.sha512('intkey'.encode('utf-8')).hexdigest()[0:6] addr = intkey_prefix + \ hashlib.sha512(payload["Name"].encode('utf-8')).hexdigest() payload_encode = hashlib.sha512(cbor.dumps(payload)).hexdigest() header = TransactionHeader( signer_pubkey=self.public_key, family_name='intkey', family_version='1.0', inputs=[addr], outputs=[addr], dependencies=[], payload_encoding="application/cbor", payload_sha512=payload_encode) if valid_batcher: header.batcher_pubkey = self.public_key else: header.batcher_pubkey = "bad_batcher" header_bytes = header.SerializeToString() if valid_signature: signature = signing.sign( header_bytes, self.private_key) else: signature = "bad_signature" if not matched_payload: payload['Name'] = 'unmatched_payload' transaction = Transaction( header=header_bytes, payload=cbor.dumps(payload), header_signature=signature) txn_list.append(transaction) return txn_list
def validate_transaction(txn): # validate transactions signature header = TransactionHeader() header.ParseFromString(txn.header) valid = signing.verify(txn.header, txn.header_signature, header.signer_pubkey) if not valid: LOGGER.debug("transaction signature invalid for txn: %s", txn.header_signature) return valid
def _create_transactions(self, count, matched_payload=True, valid_signature=True, valid_batcher=True): txn_list = [] for i in range(count): payload = {'Verb': 'set', 'Name': 'name' + str(random.randint(0, 100)), 'Value': random.randint(0, 100)} intkey_prefix = \ hashlib.sha512('intkey'.encode('utf-8')).hexdigest()[0:6] addr = intkey_prefix + \ hashlib.sha512(payload["Name"].encode('utf-8')).hexdigest() payload_encode = hashlib.sha512(cbor.dumps(payload)).hexdigest() header = TransactionHeader( signer_pubkey=self.public_key, family_name='intkey', family_version='1.0', inputs=[addr], outputs=[addr], dependencies=[], payload_encoding="application/cbor", payload_sha512=payload_encode) if valid_batcher: header.batcher_pubkey = self.public_key else: header.batcher_pubkey = "bad_batcher" header_bytes = header.SerializeToString() if valid_signature: signature = signing.sign( header_bytes, self.private_key) else: signature = "bad_signature" if not matched_payload: payload['Name'] = 'unmatched_payload' transaction = Transaction( header=header_bytes, payload=cbor.dumps(payload), header_signature=signature) txn_list.append(transaction) return txn_list
def _check_transaction_dependencies(self, txn, committed_txn): """Check that all this transactions dependencies are present. :param tx: the transaction to check :param committed_txn(TransactionCache): Current set of committed :return: Boolean, True if dependencies checkout, False otherwise. """ txn_hdr = TransactionHeader() txn_hdr.ParseFromString(txn.header) for dep in txn_hdr.dependencies: if dep not in committed_txn: LOGGER.debug("Transaction rejected due " + "missing dependency, transaction " + "%s depends on %s", txn.header_signature, dep) return False return True
def check_off_chain_transaction_roles(self, transactions): """ Check the transaction signing key against the allowed off-chain transactor permissions. The roles being checked are the following, from first to last: "transactor.transaction_signer.<TP_Name>" "transactor.transaction_signer" "transactor" The first role that is set will be the one used to enforce if the transaction signer is allowed. Args: transactions (List of Transactions): The transactions that are being verified. identity_view (IdentityView): The IdentityView that should be used to verify the transactions. """ policy = None if "transactor.transaction_signer" in self._permissions: policy = self._permissions["transactor.transaction_signer"] elif "transactor" in self._permissions: policy = self._permissions["transactor"] for transaction in transactions: header = TransactionHeader() header.ParseFromString(transaction.header) family_role = "transactor.transaction_signer." + \ header.family_name family_policy = None if family_role in self._permissions: family_policy = self._permissions[family_role] if family_policy is not None: if not self._allowed(header.signer_pubkey, family_policy): LOGGER.debug( "Transaction Signer: %s is not permitted" "by local configuration.", header.signer_pubkey) return False elif policy is not None: if not self._allowed(header.signer_pubkey, policy): LOGGER.debug( "Transaction Signer: %s is not permitted" "by local configuration.", header.signer_pubkey) return False return True
def create_batch(self, block_info): payload = BlockInfoTxn(block=block_info).SerializeToString() public_key = self._signer.get_public_key().as_hex() header = TransactionHeader( signer_public_key=public_key, family_name=FAMILY_NAME, family_version=FAMILY_VERSION, inputs=[CONFIG_ADDRESS, BLOCK_INFO_NAMESPACE], outputs=[CONFIG_ADDRESS, BLOCK_INFO_NAMESPACE], dependencies=[], payload_sha512=hashlib.sha512(payload).hexdigest(), batcher_public_key=public_key, ).SerializeToString() transaction_signature = self._signer.sign(header) transaction = Transaction( header=header, payload=payload, header_signature=transaction_signature, ) header = BatchHeader( signer_public_key=public_key, transaction_ids=[transaction_signature], ).SerializeToString() batch_signature = self._signer.sign(header) return Batch( header=header, transactions=[transaction], header_signature=batch_signature, )
def _make_block(self, txns_family, signer_pubkey, same_pubkey=True): transactions = [] for family in txns_family: txn_header = TransactionHeader( family_name=family, signer_pubkey=signer_pubkey) txn = Transaction(header=txn_header.SerializeToString()) transactions.append(txn) batch = Batch(transactions=transactions) if same_pubkey: block_header = BlockHeader(signer_pubkey=signer_pubkey) else: block_header = BlockHeader(signer_pubkey="other") block = Block(header=block_header.SerializeToString(), batches=[batch]) return BlockWrapper(block)
def _create_transactions(self, count, missing_dep=False): txn_list = [] for _ in range(count): payload = { 'Verb': 'set', 'Name': 'name' + str(random.randint(0, 100)), 'Value': random.randint(0, 100) } intkey_prefix = \ hashlib.sha512('intkey'.encode('utf-8')).hexdigest()[0:6] addr = intkey_prefix + \ hashlib.sha512(payload["Name"].encode('utf-8')).hexdigest() payload_encode = hashlib.sha512(cbor.dumps(payload)).hexdigest() header = TransactionHeader( signer_public_key=self.signer.get_public_key().as_hex(), family_name='intkey', family_version='1.0', inputs=[addr], outputs=[addr], dependencies=[], batcher_public_key=self.signer.get_public_key().as_hex(), payload_sha512=payload_encode) if missing_dep: header.dependencies.extend(["Missing"]) header_bytes = header.SerializeToString() signature = self.signer.sign(header_bytes) transaction = Transaction( header=header_bytes, payload=cbor.dumps(payload), header_signature=signature) txn_list.append(transaction) return txn_list
def is_valid_transaction(txn): # validate transactions signature header = TransactionHeader() header.ParseFromString(txn.header) if not signing.verify(txn.header, txn.header_signature, header.signer_public_key): LOGGER.debug("transaction signature invalid for txn: %s", txn.header_signature) return False # verify the payload field matches the header txn_payload_sha512 = hashlib.sha512(txn.payload).hexdigest() if txn_payload_sha512 != header.payload_sha512: LOGGER.debug( "payload doesn't match payload_sha512 of the header" "for txn: %s", txn.header_signature) return False return True
def _create_transactions(self, count): txn_list = [] for _ in range(count): payload = { 'Verb': 'set', 'Name': 'name', 'Value': 1, } intkey_prefix = \ hashlib.sha512('intkey'.encode('utf-8')).hexdigest()[0:6] addr = intkey_prefix + \ hashlib.sha512(payload["Name"].encode('utf-8')).hexdigest() payload_encode = hashlib.sha512(cbor.dumps(payload)).hexdigest() header = TransactionHeader( signer_public_key=self.public_key, family_name='intkey', family_version='1.0', inputs=[addr], outputs=[addr], dependencies=[], payload_sha512=payload_encode) header.batcher_public_key = self.public_key header_bytes = header.SerializeToString() signature = self.signer.sign(header_bytes) transaction = Transaction( header=header_bytes, payload=cbor.dumps(payload), header_signature=signature) txn_list.append(transaction) return txn_list