def mark_input_as_used(self, tx: Transaction, txin: TxInput) -> None: """ Mark a given input as used """ assert tx.hash is not None assert tx.storage is not None spent_tx = tx.storage.get_transaction(txin.tx_id) spent_meta = spent_tx.get_metadata() spent_by = spent_meta.spent_outputs[txin.index] # Set[bytes(hash)] assert tx.hash not in spent_by # Update our meta.conflict_with. meta = tx.get_metadata() if spent_by: # We initially void ourselves. This conflict will be resolved later. if not meta.voided_by: meta.voided_by = {tx.hash} else: meta.voided_by.add(tx.hash) if meta.conflict_with: meta.conflict_with.extend(spent_by) else: meta.conflict_with = spent_by.copy() tx.storage.save_transaction(tx, only_metadata=True) for h in spent_by: # Update meta.conflict_with of our conflict transactions. conflict_tx = tx.storage.get_transaction(h) tx_meta = conflict_tx.get_metadata() if tx_meta.conflict_with: tx_meta.conflict_with.append(tx.hash) else: tx_meta.conflict_with = [tx.hash] tx.storage.save_transaction(conflict_tx, only_metadata=True) # Add ourselves to meta.spent_by of our input. spent_by.append(tx.hash) tx.storage.save_transaction(spent_tx, only_metadata=True)
class BaseTransactionStorageTest(unittest.TestCase): __test__ = False def setUp(self, tx_storage, reactor=None): from hathor.manager import HathorManager if not reactor: self.reactor = Clock() else: self.reactor = reactor self.reactor.advance(time.time()) self.tx_storage = tx_storage assert tx_storage.first_timestamp > 0 tx_storage._manually_initialize() self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] self.tmpdir = tempfile.mkdtemp() wallet = Wallet(directory=self.tmpdir) wallet.unlock(b'teste') self.manager = HathorManager(self.reactor, tx_storage=self.tx_storage, wallet=wallet) self.tx_storage.indexes.enable_address_index(self.manager.pubsub) self.tx_storage.indexes.enable_tokens_index() block_parents = [ tx.hash for tx in chain(self.genesis_blocks, self.genesis_txs) ] output = TxOutput(200, P2PKH.create_output_script(BURN_ADDRESS)) self.block = Block(timestamp=MIN_TIMESTAMP, weight=12, outputs=[output], parents=block_parents, nonce=100781, storage=tx_storage) self.block.resolve() self.block.verify() self.block.get_metadata().validation = ValidationState.FULL tx_parents = [tx.hash for tx in self.genesis_txs] tx_input = TxInput( tx_id=self.genesis_blocks[0].hash, index=0, data=bytes.fromhex( '46304402203470cb9818c9eb842b0c433b7e2b8aded0a51f5903e971649e870763d0266a' 'd2022049b48e09e718c4b66a0f3178ef92e4d60ee333d2d0e25af8868acf5acbb35aaa583' '056301006072a8648ce3d020106052b8104000a034200042ce7b94cba00b654d4308f8840' '7345cacb1f1032fb5ac80407b74d56ed82fb36467cb7048f79b90b1cf721de57e942c5748' '620e78362cf2d908e9057ac235a63')) self.tx = Transaction( timestamp=MIN_TIMESTAMP + 2, weight=10, nonce=932049, inputs=[tx_input], outputs=[output], tokens=[ bytes.fromhex( '0023be91834c973d6a6ddd1a0ae411807b7c8ef2a015afb5177ee64b666ce602' ) ], parents=tx_parents, storage=tx_storage) self.tx.resolve() self.tx.get_metadata().validation = ValidationState.FULL # Disable weakref to test the internal methods. Otherwise, most methods return objects from weakref. self.tx_storage._disable_weakref() self.tx_storage.enable_lock() def tearDown(self): shutil.rmtree(self.tmpdir) def test_genesis_ref(self): # Enable weakref to this test only. self.tx_storage._enable_weakref() genesis_set = set(self.tx_storage.get_all_genesis()) for tx in genesis_set: tx2 = self.tx_storage.get_transaction(tx.hash) self.assertTrue(tx is tx2) from hathor.transaction.genesis import _get_genesis_transactions_unsafe genesis_from_settings = _get_genesis_transactions_unsafe(None) for tx in genesis_from_settings: tx2 = self.tx_storage.get_transaction(tx.hash) self.assertTrue(tx is not tx2) for tx3 in genesis_set: self.assertTrue(tx is not tx3) if tx2 == tx3: self.assertTrue(tx2 is tx3) def test_genesis(self): self.assertEqual(1, len(self.genesis_blocks)) self.assertEqual(2, len(self.genesis_txs)) for tx in self.genesis: tx.verify() for tx in self.genesis: tx2 = self.tx_storage.get_transaction(tx.hash) self.assertEqual(tx, tx2) self.assertTrue(self.tx_storage.transaction_exists(tx.hash)) def test_get_empty_merklee_tree(self): # We use `first_timestamp - 1` to ensure that the merkle tree will be empty. self.tx_storage.get_merkle_tree(self.tx_storage.first_timestamp - 1) def test_first_timestamp(self): self.assertEqual(self.tx_storage.first_timestamp, min(x.timestamp for x in self.genesis)) def test_storage_basic(self): self.assertEqual(1, self.tx_storage.get_block_count()) self.assertEqual(2, self.tx_storage.get_tx_count()) self.assertEqual(3, self.tx_storage.get_count_tx_blocks()) block_parents_hash = [x.data for x in self.tx_storage.get_block_tips()] self.assertEqual(1, len(block_parents_hash)) self.assertEqual(block_parents_hash, [self.genesis_blocks[0].hash]) tx_parents_hash = [x.data for x in self.tx_storage.get_tx_tips()] self.assertEqual(2, len(tx_parents_hash)) self.assertEqual(set(tx_parents_hash), {self.genesis_txs[0].hash, self.genesis_txs[1].hash}) def test_storage_basic_v2(self): self.assertEqual(1, self.tx_storage.get_block_count()) self.assertEqual(2, self.tx_storage.get_tx_count()) self.assertEqual(3, self.tx_storage.get_count_tx_blocks()) block_parents_hash = self.tx_storage.get_best_block_tips() self.assertEqual(1, len(block_parents_hash)) self.assertEqual(block_parents_hash, [self.genesis_blocks[0].hash]) tx_parents_hash = self.manager.get_new_tx_parents() self.assertEqual(2, len(tx_parents_hash)) self.assertEqual(set(tx_parents_hash), {self.genesis_txs[0].hash, self.genesis_txs[1].hash}) def validate_save(self, obj): self.tx_storage.save_transaction(obj, add_to_indexes=True) loaded_obj1 = self.tx_storage.get_transaction(obj.hash) self.assertTrue(self.tx_storage.transaction_exists(obj.hash)) self.assertEqual(obj, loaded_obj1) self.assertEqual(len(obj.get_funds_struct()), len(loaded_obj1.get_funds_struct())) self.assertEqual(bytes(obj), bytes(loaded_obj1)) self.assertEqual(obj.to_json(), loaded_obj1.to_json()) self.assertEqual(obj.is_block, loaded_obj1.is_block) # Testing add and remove from cache if self.tx_storage.with_index: if obj.is_block: self.assertTrue(obj.hash in self.tx_storage.indexes.block_tips. tx_last_interval) else: self.assertTrue(obj.hash in self.tx_storage.indexes.tx_tips. tx_last_interval) self.tx_storage.del_from_indexes(obj) if self.tx_storage.with_index: if obj.is_block: self.assertFalse(obj.hash in self.tx_storage.indexes. block_tips.tx_last_interval) else: self.assertFalse(obj.hash in self.tx_storage.indexes.tx_tips. tx_last_interval) self.tx_storage.add_to_indexes(obj) if self.tx_storage.with_index: if obj.is_block: self.assertTrue(obj.hash in self.tx_storage.indexes.block_tips. tx_last_interval) else: self.assertTrue(obj.hash in self.tx_storage.indexes.tx_tips. tx_last_interval) def test_save_block(self): self.validate_save(self.block) def test_save_tx(self): self.validate_save(self.tx) def test_save_token_creation_tx(self): tx = create_tokens(self.manager, propagate=False) tx.get_metadata().validation = ValidationState.FULL self.validate_save(tx) def _validate_not_in_index(self, tx, index): tips = index.tips_index[self.tx.timestamp] self.assertNotIn(self.tx.hash, [x.data for x in tips]) self.assertNotIn(self.tx.hash, index.tips_index.tx_last_interval) self.assertIsNone(index.txs_index.find_tx_index(tx)) def _test_remove_tx_or_block(self, tx): self.validate_save(tx) self.tx_storage.remove_transaction(tx) with self.assertRaises(TransactionDoesNotExist): self.tx_storage.get_transaction(tx.hash) if hasattr(self.tx_storage, 'all_index'): self._validate_not_in_index(tx, self.tx_storage.all_index) if tx.is_block: if hasattr(self.tx_storage, 'block_index'): self._validate_not_in_index(tx, self.tx_storage.block_index) else: if hasattr(self.tx_storage, 'tx_index'): self._validate_not_in_index(tx, self.tx_storage.tx_index) # Check wallet index. addresses_index = self.tx_storage.indexes.addresses addresses = tx.get_related_addresses() for address in addresses: self.assertNotIn(tx.hash, addresses_index.get_from_address(address)) # TODO Check self.tx_storage.tokens_index # Try to remove twice. It is supposed to do nothing. self.tx_storage.remove_transaction(tx) def test_remove_tx(self): self._test_remove_tx_or_block(self.tx) def test_remove_block(self): self._test_remove_tx_or_block(self.block) def test_shared_memory(self): # Enable weakref to this test only. self.tx_storage._enable_weakref() self.validate_save(self.block) self.validate_save(self.tx) for tx in [self.tx, self.block]: # just making sure, if it is genesis the test is wrong self.assertFalse(tx.is_genesis) # load transactions twice tx1 = self.tx_storage.get_transaction(tx.hash) tx2 = self.tx_storage.get_transaction(tx.hash) # naturally they should be equal, but this time so do the objects self.assertTrue(tx1 == tx2) self.assertTrue(tx1 is tx2) meta1 = tx1.get_metadata() meta2 = tx2.get_metadata() # and naturally the metadata too self.assertTrue(meta1 == meta2) self.assertTrue(meta1 is meta2) def test_get_wrong_tx(self): hex_error = bytes.fromhex( '00001c5c0b69d13b05534c94a69b2c8272294e6b0c536660a3ac264820677024') with self.assertRaises(TransactionDoesNotExist): self.tx_storage.get_transaction(hex_error) def test_save_metadata(self): # Saving genesis metadata self.tx_storage.save_transaction(self.genesis_txs[0], only_metadata=True) tx = self.block # First we save to the storage self.tx_storage.save_transaction(tx) metadata = tx.get_metadata() metadata.spent_outputs[1].append(self.genesis_blocks[0].hash) random_tx = bytes.fromhex( '0000222e64683b966b4268f387c269915cc61f6af5329823a93e3696cb0f2222') metadata.children.append(random_tx) self.tx_storage.save_transaction(tx, only_metadata=True) tx2 = self.tx_storage.get_transaction(tx.hash) metadata2 = tx2.get_metadata() self.assertEqual(metadata, metadata2) total = 0 for tx in self.tx_storage.get_all_transactions(): total += 1 self.assertEqual(total, 4) def test_storage_new_blocks(self): tip_blocks = [x.data for x in self.tx_storage.get_block_tips()] self.assertEqual(tip_blocks, [self.genesis_blocks[0].hash]) block1 = self._add_new_block() tip_blocks = [x.data for x in self.tx_storage.get_block_tips()] self.assertEqual(tip_blocks, [block1.hash]) block2 = self._add_new_block() tip_blocks = [x.data for x in self.tx_storage.get_block_tips()] self.assertEqual(tip_blocks, [block2.hash]) # Block3 has the same parents as block2. block3 = self._add_new_block(parents=block2.parents) tip_blocks = [x.data for x in self.tx_storage.get_block_tips()] self.assertEqual(set(tip_blocks), {block2.hash, block3.hash}) # Re-generate caches to test topological sort. self.tx_storage._manually_initialize() tip_blocks = [x.data for x in self.tx_storage.get_block_tips()] self.assertEqual(set(tip_blocks), {block2.hash, block3.hash}) def test_token_list(self): tx = self.tx self.validate_save(tx) # 2 token uids tx.tokens.append( bytes.fromhex( '00001c5c0b69d13b05534c94a69b2c8272294e6b0c536660a3ac264820677024' )) tx.resolve() self.validate_save(tx) # no tokens tx.tokens = [] tx.resolve() self.validate_save(tx) def _add_new_block(self, parents=None): block = self.manager.generate_mining_block() block.data = b'Testing, testing, 1, 2, 3... testing, testing...' if parents is not None: block.parents = parents block.weight = 10 self.assertTrue(block.resolve()) block.verify() self.manager.propagate_tx(block, fails_silently=False) self.reactor.advance(5) return block def test_topological_sort(self): _set_test_mode(TestMode.TEST_ALL_WEIGHT) _total = 0 blocks = add_new_blocks(self.manager, 1, advance_clock=1) _total += len(blocks) blocks = add_blocks_unlock_reward(self.manager) _total += len(blocks) add_new_transactions(self.manager, 1, advance_clock=1) total = 0 for tx in self.tx_storage._topological_sort(): total += 1 # added blocks + genesis txs + added tx self.assertEqual(total, _total + 3 + 1) def test_get_best_block_weight(self): block = self._add_new_block() weight = self.tx_storage.get_weight_best_block() self.assertEqual(block.weight, weight) @inlineCallbacks def test_concurrent_access(self): self.tx_storage.save_transaction(self.tx) self.tx_storage._enable_weakref() def handle_error(err): self.fail( 'Error resolving concurrent access deferred. {}'.format(err)) deferreds = [] for i in range(5): d = deferToThread(self.tx_storage.get_transaction, self.tx.hash) d.addErrback(handle_error) deferreds.append(d) self.reactor.advance(3) yield gatherResults(deferreds) self.tx_storage._disable_weakref() def test_full_verification_attribute(self): self.assertFalse(self.tx_storage.is_running_full_verification()) self.tx_storage.start_full_verification() self.assertTrue(self.tx_storage.is_running_full_verification()) self.tx_storage.finish_full_verification() self.assertFalse(self.tx_storage.is_running_full_verification()) def test_key_value_attribute(self): attr = 'test' val = 'a' # Try to get a key that does not exist self.assertIsNone(self.tx_storage.get_value(attr)) # Try to remove this key that does not exist self.tx_storage.remove_value(attr) # Add the key/value self.tx_storage.add_value(attr, val) # Get correct value self.assertEqual(self.tx_storage.get_value(attr), val) # Remove the key self.tx_storage.remove_value(attr) # Key should not exist again self.assertIsNone(self.tx_storage.get_value(attr))
def check_conflicts(self, tx: Transaction) -> None: """ Check which transaction is the winner of a conflict, the remaining are voided. The verification is made for each input, and `self` is only marked as winner if it wins in all its inputs. """ assert tx.hash is not None assert tx.storage is not None self.log.debug('tx.check_conflicts', tx=tx.hash_hex) meta = tx.get_metadata() if meta.voided_by != {tx.hash}: return # Filter the possible candidates to compare to tx. candidates: List[Transaction] = [] conflict_list: List[Transaction] = [] for h in meta.conflict_with or []: conflict_tx = cast(Transaction, tx.storage.get_transaction(h)) conflict_list.append(conflict_tx) conflict_tx_meta = conflict_tx.get_metadata() if not conflict_tx_meta.voided_by or conflict_tx_meta.voided_by == { conflict_tx.hash }: candidates.append(conflict_tx) # Check whether we have the highest accumulated weight. # First with the voided transactions. is_highest = True for candidate in candidates: tx_meta = candidate.get_metadata() if tx_meta.voided_by: if tx_meta.accumulated_weight > meta.accumulated_weight: is_highest = False break if not is_highest: return # Then, with the executed transactions. tie_list = [] for candidate in candidates: tx_meta = candidate.get_metadata() if not tx_meta.voided_by: candidate.update_accumulated_weight( stop_value=meta.accumulated_weight) tx_meta = candidate.get_metadata() d = tx_meta.accumulated_weight - meta.accumulated_weight if abs(d) < settings.WEIGHT_TOL: tie_list.append(candidate) elif d > 0: is_highest = False break if not is_highest: return # If we got here, either it was a tie or we won. # So, let's void the conflict txs. for conflict_tx in conflict_list: self.mark_as_voided(conflict_tx) if not tie_list: # If it is not a tie, we won. \o/ self.mark_as_winner(tx)
def update_voided_info(self, tx: Transaction) -> None: """ This method should be called only once when the transactions is added to the DAG. """ assert tx.hash is not None assert tx.storage is not None voided_by: Set[bytes] = set() # Union of voided_by of parents for parent in tx.get_parents(): parent_meta = parent.get_metadata() if parent_meta.voided_by: voided_by.update( self.consensus.filter_out_soft_voided_entries( parent, parent_meta.voided_by)) assert settings.SOFT_VOIDED_ID not in voided_by assert not (self.soft_voided_tx_ids & voided_by) # Union of voided_by of inputs for txin in tx.inputs: spent_tx = tx.storage.get_transaction(txin.tx_id) spent_meta = spent_tx.get_metadata() if spent_meta.voided_by: voided_by.update(spent_meta.voided_by) voided_by.discard(settings.SOFT_VOIDED_ID) assert settings.SOFT_VOIDED_ID not in voided_by # Update accumulated weight of the transactions voiding us. assert tx.hash not in voided_by for h in voided_by: if h == settings.SOFT_VOIDED_ID: continue tx2 = tx.storage.get_transaction(h) tx2_meta = tx2.get_metadata() tx2_meta.accumulated_weight = sum_weights( tx2_meta.accumulated_weight, tx.weight) assert tx2.storage is not None tx2.storage.save_transaction(tx2, only_metadata=True) # Then, we add ourselves. meta = tx.get_metadata() assert not meta.voided_by or meta.voided_by == {tx.hash} assert meta.accumulated_weight == tx.weight if tx.hash in self.soft_voided_tx_ids: voided_by.add(settings.SOFT_VOIDED_ID) voided_by.add(tx.hash) if meta.conflict_with: voided_by.add(tx.hash) # We must save before marking conflicts as voided because # the conflicting tx might affect this tx's voided_by metadata. if voided_by: meta.voided_by = voided_by.copy() tx.storage.save_transaction(tx, only_metadata=True) tx.storage.del_from_indexes(tx) # Check conflicts of the transactions voiding us. for h in voided_by: if h == settings.SOFT_VOIDED_ID: continue if h == tx.hash: continue tx2 = tx.storage.get_transaction(h) if not tx2.is_block: assert isinstance(tx2, Transaction) self.check_conflicts(tx2) # Mark voided conflicts as voided. for h in meta.conflict_with or []: conflict_tx = cast(Transaction, tx.storage.get_transaction(h)) conflict_tx_meta = conflict_tx.get_metadata() if conflict_tx_meta.voided_by: self.mark_as_voided(conflict_tx) # Finally, check our conflicts. meta = tx.get_metadata() if meta.voided_by == {tx.hash}: self.check_conflicts(tx) # Assert the final state is valid. self.assert_valid_consensus(tx)
def check_twins(self, tx: Transaction, transactions: Iterable[BaseTransaction]) -> None: """ Check if the tx has any twins in transactions list A twin tx is a tx that has the same inputs and outputs We add all the hashes of the twin txs in the metadata :param transactions: list of transactions to be checked if they are twins with self """ assert tx.hash is not None assert tx.storage is not None # Getting tx metadata to save the new twins meta = tx.get_metadata() # Sorting inputs and outputs for easier validation sorted_inputs = sorted(tx.inputs, key=lambda x: (x.tx_id, x.index, x.data)) sorted_outputs = sorted(tx.outputs, key=lambda x: (x.script, x.value)) for candidate in transactions: assert candidate.hash is not None # If quantity of inputs is different, it's not a twin. if len(candidate.inputs) != len(tx.inputs): continue # If quantity of outputs is different, it's not a twin. if len(candidate.outputs) != len(tx.outputs): continue # If the hash is the same, it's not a twin. if candidate.hash == tx.hash: continue # Verify if all the inputs are the same equal = True for index, tx_input in enumerate( sorted(candidate.inputs, key=lambda x: (x.tx_id, x.index, x.data))): if (tx_input.tx_id != sorted_inputs[index].tx_id or tx_input.data != sorted_inputs[index].data or tx_input.index != sorted_inputs[index].index): equal = False break # Verify if all the outputs are the same if equal: for index, tx_output in enumerate( sorted(candidate.outputs, key=lambda x: (x.script, x.value))): if (tx_output.value != sorted_outputs[index].value or tx_output.script != sorted_outputs[index].script): equal = False break # If everything is equal we add in both metadatas if equal: meta.twins.append(candidate.hash) tx_meta = candidate.get_metadata() tx_meta.twins.append(tx.hash) tx.storage.save_transaction(candidate, only_metadata=True) tx.storage.save_transaction(tx, only_metadata=True)