def mark_input_as_used(self, tx: Transaction, txin: TxInput) -> None:
""" Mark a given input as used
"""
assert tx.hash is not None
assert tx.storage is not None
spent_tx = tx.storage.get_transaction(txin.tx_id)
spent_meta = spent_tx.get_metadata()
spent_by = spent_meta.spent_outputs[txin.index] # Set[bytes(hash)]
assert tx.hash not in spent_by
# Update our meta.conflict_with.
meta = tx.get_metadata()
if spent_by:
# We initially void ourselves. This conflict will be resolved later.
if not meta.voided_by:
meta.voided_by = {tx.hash}
else:
meta.voided_by.add(tx.hash)
if meta.conflict_with:
meta.conflict_with.extend(spent_by)
else:
meta.conflict_with = spent_by.copy()
tx.storage.save_transaction(tx, only_metadata=True)
for h in spent_by:
# Update meta.conflict_with of our conflict transactions.
conflict_tx = tx.storage.get_transaction(h)
tx_meta = conflict_tx.get_metadata()
if tx_meta.conflict_with:
tx_meta.conflict_with.append(tx.hash)
else:
tx_meta.conflict_with = [tx.hash]
tx.storage.save_transaction(conflict_tx, only_metadata=True)
# Add ourselves to meta.spent_by of our input.
spent_by.append(tx.hash)
tx.storage.save_transaction(spent_tx, only_metadata=True)
class BaseTransactionStorageTest(unittest.TestCase):
__test__ = False
def setUp(self, tx_storage, reactor=None):
from hathor.manager import HathorManager
if not reactor:
self.reactor = Clock()
else:
self.reactor = reactor
self.reactor.advance(time.time())
self.tx_storage = tx_storage
assert tx_storage.first_timestamp > 0
tx_storage._manually_initialize()
self.genesis = self.tx_storage.get_all_genesis()
self.genesis_blocks = [tx for tx in self.genesis if tx.is_block]
self.genesis_txs = [tx for tx in self.genesis if not tx.is_block]
self.tmpdir = tempfile.mkdtemp()
wallet = Wallet(directory=self.tmpdir)
wallet.unlock(b'teste')
self.manager = HathorManager(self.reactor,
tx_storage=self.tx_storage,
wallet=wallet)
self.tx_storage.indexes.enable_address_index(self.manager.pubsub)
self.tx_storage.indexes.enable_tokens_index()
block_parents = [
tx.hash for tx in chain(self.genesis_blocks, self.genesis_txs)
]
output = TxOutput(200, P2PKH.create_output_script(BURN_ADDRESS))
self.block = Block(timestamp=MIN_TIMESTAMP,
weight=12,
outputs=[output],
parents=block_parents,
nonce=100781,
storage=tx_storage)
self.block.resolve()
self.block.verify()
self.block.get_metadata().validation = ValidationState.FULL
tx_parents = [tx.hash for tx in self.genesis_txs]
tx_input = TxInput(
tx_id=self.genesis_blocks[0].hash,
index=0,
data=bytes.fromhex(
'46304402203470cb9818c9eb842b0c433b7e2b8aded0a51f5903e971649e870763d0266a'
'd2022049b48e09e718c4b66a0f3178ef92e4d60ee333d2d0e25af8868acf5acbb35aaa583'
'056301006072a8648ce3d020106052b8104000a034200042ce7b94cba00b654d4308f8840'
'7345cacb1f1032fb5ac80407b74d56ed82fb36467cb7048f79b90b1cf721de57e942c5748'
'620e78362cf2d908e9057ac235a63'))
self.tx = Transaction(
timestamp=MIN_TIMESTAMP + 2,
weight=10,
nonce=932049,
inputs=[tx_input],
outputs=[output],
tokens=[
bytes.fromhex(
'0023be91834c973d6a6ddd1a0ae411807b7c8ef2a015afb5177ee64b666ce602'
)
],
parents=tx_parents,
storage=tx_storage)
self.tx.resolve()
self.tx.get_metadata().validation = ValidationState.FULL
# Disable weakref to test the internal methods. Otherwise, most methods return objects from weakref.
self.tx_storage._disable_weakref()
self.tx_storage.enable_lock()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_genesis_ref(self):
# Enable weakref to this test only.
self.tx_storage._enable_weakref()
genesis_set = set(self.tx_storage.get_all_genesis())
for tx in genesis_set:
tx2 = self.tx_storage.get_transaction(tx.hash)
self.assertTrue(tx is tx2)
from hathor.transaction.genesis import _get_genesis_transactions_unsafe
genesis_from_settings = _get_genesis_transactions_unsafe(None)
for tx in genesis_from_settings:
tx2 = self.tx_storage.get_transaction(tx.hash)
self.assertTrue(tx is not tx2)
for tx3 in genesis_set:
self.assertTrue(tx is not tx3)
if tx2 == tx3:
self.assertTrue(tx2 is tx3)
def test_genesis(self):
self.assertEqual(1, len(self.genesis_blocks))
self.assertEqual(2, len(self.genesis_txs))
for tx in self.genesis:
tx.verify()
for tx in self.genesis:
tx2 = self.tx_storage.get_transaction(tx.hash)
self.assertEqual(tx, tx2)
self.assertTrue(self.tx_storage.transaction_exists(tx.hash))
def test_get_empty_merklee_tree(self):
# We use `first_timestamp - 1` to ensure that the merkle tree will be empty.
self.tx_storage.get_merkle_tree(self.tx_storage.first_timestamp - 1)
def test_first_timestamp(self):
self.assertEqual(self.tx_storage.first_timestamp,
min(x.timestamp for x in self.genesis))
def test_storage_basic(self):
self.assertEqual(1, self.tx_storage.get_block_count())
self.assertEqual(2, self.tx_storage.get_tx_count())
self.assertEqual(3, self.tx_storage.get_count_tx_blocks())
block_parents_hash = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(1, len(block_parents_hash))
self.assertEqual(block_parents_hash, [self.genesis_blocks[0].hash])
tx_parents_hash = [x.data for x in self.tx_storage.get_tx_tips()]
self.assertEqual(2, len(tx_parents_hash))
self.assertEqual(set(tx_parents_hash),
{self.genesis_txs[0].hash, self.genesis_txs[1].hash})
def test_storage_basic_v2(self):
self.assertEqual(1, self.tx_storage.get_block_count())
self.assertEqual(2, self.tx_storage.get_tx_count())
self.assertEqual(3, self.tx_storage.get_count_tx_blocks())
block_parents_hash = self.tx_storage.get_best_block_tips()
self.assertEqual(1, len(block_parents_hash))
self.assertEqual(block_parents_hash, [self.genesis_blocks[0].hash])
tx_parents_hash = self.manager.get_new_tx_parents()
self.assertEqual(2, len(tx_parents_hash))
self.assertEqual(set(tx_parents_hash),
{self.genesis_txs[0].hash, self.genesis_txs[1].hash})
def validate_save(self, obj):
self.tx_storage.save_transaction(obj, add_to_indexes=True)
loaded_obj1 = self.tx_storage.get_transaction(obj.hash)
self.assertTrue(self.tx_storage.transaction_exists(obj.hash))
self.assertEqual(obj, loaded_obj1)
self.assertEqual(len(obj.get_funds_struct()),
len(loaded_obj1.get_funds_struct()))
self.assertEqual(bytes(obj), bytes(loaded_obj1))
self.assertEqual(obj.to_json(), loaded_obj1.to_json())
self.assertEqual(obj.is_block, loaded_obj1.is_block)
# Testing add and remove from cache
if self.tx_storage.with_index:
if obj.is_block:
self.assertTrue(obj.hash in self.tx_storage.indexes.block_tips.
tx_last_interval)
else:
self.assertTrue(obj.hash in self.tx_storage.indexes.tx_tips.
tx_last_interval)
self.tx_storage.del_from_indexes(obj)
if self.tx_storage.with_index:
if obj.is_block:
self.assertFalse(obj.hash in self.tx_storage.indexes.
block_tips.tx_last_interval)
else:
self.assertFalse(obj.hash in self.tx_storage.indexes.tx_tips.
tx_last_interval)
self.tx_storage.add_to_indexes(obj)
if self.tx_storage.with_index:
if obj.is_block:
self.assertTrue(obj.hash in self.tx_storage.indexes.block_tips.
tx_last_interval)
else:
self.assertTrue(obj.hash in self.tx_storage.indexes.tx_tips.
tx_last_interval)
def test_save_block(self):
self.validate_save(self.block)
def test_save_tx(self):
self.validate_save(self.tx)
def test_save_token_creation_tx(self):
tx = create_tokens(self.manager, propagate=False)
tx.get_metadata().validation = ValidationState.FULL
self.validate_save(tx)
def _validate_not_in_index(self, tx, index):
tips = index.tips_index[self.tx.timestamp]
self.assertNotIn(self.tx.hash, [x.data for x in tips])
self.assertNotIn(self.tx.hash, index.tips_index.tx_last_interval)
self.assertIsNone(index.txs_index.find_tx_index(tx))
def _test_remove_tx_or_block(self, tx):
self.validate_save(tx)
self.tx_storage.remove_transaction(tx)
with self.assertRaises(TransactionDoesNotExist):
self.tx_storage.get_transaction(tx.hash)
if hasattr(self.tx_storage, 'all_index'):
self._validate_not_in_index(tx, self.tx_storage.all_index)
if tx.is_block:
if hasattr(self.tx_storage, 'block_index'):
self._validate_not_in_index(tx, self.tx_storage.block_index)
else:
if hasattr(self.tx_storage, 'tx_index'):
self._validate_not_in_index(tx, self.tx_storage.tx_index)
# Check wallet index.
addresses_index = self.tx_storage.indexes.addresses
addresses = tx.get_related_addresses()
for address in addresses:
self.assertNotIn(tx.hash,
addresses_index.get_from_address(address))
# TODO Check self.tx_storage.tokens_index
# Try to remove twice. It is supposed to do nothing.
self.tx_storage.remove_transaction(tx)
def test_remove_tx(self):
self._test_remove_tx_or_block(self.tx)
def test_remove_block(self):
self._test_remove_tx_or_block(self.block)
def test_shared_memory(self):
# Enable weakref to this test only.
self.tx_storage._enable_weakref()
self.validate_save(self.block)
self.validate_save(self.tx)
for tx in [self.tx, self.block]:
# just making sure, if it is genesis the test is wrong
self.assertFalse(tx.is_genesis)
# load transactions twice
tx1 = self.tx_storage.get_transaction(tx.hash)
tx2 = self.tx_storage.get_transaction(tx.hash)
# naturally they should be equal, but this time so do the objects
self.assertTrue(tx1 == tx2)
self.assertTrue(tx1 is tx2)
meta1 = tx1.get_metadata()
meta2 = tx2.get_metadata()
# and naturally the metadata too
self.assertTrue(meta1 == meta2)
self.assertTrue(meta1 is meta2)
def test_get_wrong_tx(self):
hex_error = bytes.fromhex(
'00001c5c0b69d13b05534c94a69b2c8272294e6b0c536660a3ac264820677024')
with self.assertRaises(TransactionDoesNotExist):
self.tx_storage.get_transaction(hex_error)
def test_save_metadata(self):
# Saving genesis metadata
self.tx_storage.save_transaction(self.genesis_txs[0],
only_metadata=True)
tx = self.block
# First we save to the storage
self.tx_storage.save_transaction(tx)
metadata = tx.get_metadata()
metadata.spent_outputs[1].append(self.genesis_blocks[0].hash)
random_tx = bytes.fromhex(
'0000222e64683b966b4268f387c269915cc61f6af5329823a93e3696cb0f2222')
metadata.children.append(random_tx)
self.tx_storage.save_transaction(tx, only_metadata=True)
tx2 = self.tx_storage.get_transaction(tx.hash)
metadata2 = tx2.get_metadata()
self.assertEqual(metadata, metadata2)
total = 0
for tx in self.tx_storage.get_all_transactions():
total += 1
self.assertEqual(total, 4)
def test_storage_new_blocks(self):
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(tip_blocks, [self.genesis_blocks[0].hash])
block1 = self._add_new_block()
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(tip_blocks, [block1.hash])
block2 = self._add_new_block()
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(tip_blocks, [block2.hash])
# Block3 has the same parents as block2.
block3 = self._add_new_block(parents=block2.parents)
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(set(tip_blocks), {block2.hash, block3.hash})
# Re-generate caches to test topological sort.
self.tx_storage._manually_initialize()
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(set(tip_blocks), {block2.hash, block3.hash})
def test_token_list(self):
tx = self.tx
self.validate_save(tx)
# 2 token uids
tx.tokens.append(
bytes.fromhex(
'00001c5c0b69d13b05534c94a69b2c8272294e6b0c536660a3ac264820677024'
))
tx.resolve()
self.validate_save(tx)
# no tokens
tx.tokens = []
tx.resolve()
self.validate_save(tx)
def _add_new_block(self, parents=None):
block = self.manager.generate_mining_block()
block.data = b'Testing, testing, 1, 2, 3... testing, testing...'
if parents is not None:
block.parents = parents
block.weight = 10
self.assertTrue(block.resolve())
block.verify()
self.manager.propagate_tx(block, fails_silently=False)
self.reactor.advance(5)
return block
def test_topological_sort(self):
_set_test_mode(TestMode.TEST_ALL_WEIGHT)
_total = 0
blocks = add_new_blocks(self.manager, 1, advance_clock=1)
_total += len(blocks)
blocks = add_blocks_unlock_reward(self.manager)
_total += len(blocks)
add_new_transactions(self.manager, 1, advance_clock=1)
total = 0
for tx in self.tx_storage._topological_sort():
total += 1
# added blocks + genesis txs + added tx
self.assertEqual(total, _total + 3 + 1)
def test_get_best_block_weight(self):
block = self._add_new_block()
weight = self.tx_storage.get_weight_best_block()
self.assertEqual(block.weight, weight)
@inlineCallbacks
def test_concurrent_access(self):
self.tx_storage.save_transaction(self.tx)
self.tx_storage._enable_weakref()
def handle_error(err):
self.fail(
'Error resolving concurrent access deferred. {}'.format(err))
deferreds = []
for i in range(5):
d = deferToThread(self.tx_storage.get_transaction, self.tx.hash)
d.addErrback(handle_error)
deferreds.append(d)
self.reactor.advance(3)
yield gatherResults(deferreds)
self.tx_storage._disable_weakref()
def test_full_verification_attribute(self):
self.assertFalse(self.tx_storage.is_running_full_verification())
self.tx_storage.start_full_verification()
self.assertTrue(self.tx_storage.is_running_full_verification())
self.tx_storage.finish_full_verification()
self.assertFalse(self.tx_storage.is_running_full_verification())
def test_key_value_attribute(self):
attr = 'test'
val = 'a'
# Try to get a key that does not exist
self.assertIsNone(self.tx_storage.get_value(attr))
# Try to remove this key that does not exist
self.tx_storage.remove_value(attr)
# Add the key/value
self.tx_storage.add_value(attr, val)
# Get correct value
self.assertEqual(self.tx_storage.get_value(attr), val)
# Remove the key
self.tx_storage.remove_value(attr)
# Key should not exist again
self.assertIsNone(self.tx_storage.get_value(attr))
def check_conflicts(self, tx: Transaction) -> None:
""" Check which transaction is the winner of a conflict, the remaining are voided.
The verification is made for each input, and `self` is only marked as winner if it
wins in all its inputs.
"""
assert tx.hash is not None
assert tx.storage is not None
self.log.debug('tx.check_conflicts', tx=tx.hash_hex)
meta = tx.get_metadata()
if meta.voided_by != {tx.hash}:
return
# Filter the possible candidates to compare to tx.
candidates: List[Transaction] = []
conflict_list: List[Transaction] = []
for h in meta.conflict_with or []:
conflict_tx = cast(Transaction, tx.storage.get_transaction(h))
conflict_list.append(conflict_tx)
conflict_tx_meta = conflict_tx.get_metadata()
if not conflict_tx_meta.voided_by or conflict_tx_meta.voided_by == {
conflict_tx.hash
}:
candidates.append(conflict_tx)
# Check whether we have the highest accumulated weight.
# First with the voided transactions.
is_highest = True
for candidate in candidates:
tx_meta = candidate.get_metadata()
if tx_meta.voided_by:
if tx_meta.accumulated_weight > meta.accumulated_weight:
is_highest = False
break
if not is_highest:
return
# Then, with the executed transactions.
tie_list = []
for candidate in candidates:
tx_meta = candidate.get_metadata()
if not tx_meta.voided_by:
candidate.update_accumulated_weight(
stop_value=meta.accumulated_weight)
tx_meta = candidate.get_metadata()
d = tx_meta.accumulated_weight - meta.accumulated_weight
if abs(d) < settings.WEIGHT_TOL:
tie_list.append(candidate)
elif d > 0:
is_highest = False
break
if not is_highest:
return
# If we got here, either it was a tie or we won.
# So, let's void the conflict txs.
for conflict_tx in conflict_list:
self.mark_as_voided(conflict_tx)
if not tie_list:
# If it is not a tie, we won. \o/
self.mark_as_winner(tx)
def update_voided_info(self, tx: Transaction) -> None:
""" This method should be called only once when the transactions is added to the DAG.
"""
assert tx.hash is not None
assert tx.storage is not None
voided_by: Set[bytes] = set()
# Union of voided_by of parents
for parent in tx.get_parents():
parent_meta = parent.get_metadata()
if parent_meta.voided_by:
voided_by.update(
self.consensus.filter_out_soft_voided_entries(
parent, parent_meta.voided_by))
assert settings.SOFT_VOIDED_ID not in voided_by
assert not (self.soft_voided_tx_ids & voided_by)
# Union of voided_by of inputs
for txin in tx.inputs:
spent_tx = tx.storage.get_transaction(txin.tx_id)
spent_meta = spent_tx.get_metadata()
if spent_meta.voided_by:
voided_by.update(spent_meta.voided_by)
voided_by.discard(settings.SOFT_VOIDED_ID)
assert settings.SOFT_VOIDED_ID not in voided_by
# Update accumulated weight of the transactions voiding us.
assert tx.hash not in voided_by
for h in voided_by:
if h == settings.SOFT_VOIDED_ID:
continue
tx2 = tx.storage.get_transaction(h)
tx2_meta = tx2.get_metadata()
tx2_meta.accumulated_weight = sum_weights(
tx2_meta.accumulated_weight, tx.weight)
assert tx2.storage is not None
tx2.storage.save_transaction(tx2, only_metadata=True)
# Then, we add ourselves.
meta = tx.get_metadata()
assert not meta.voided_by or meta.voided_by == {tx.hash}
assert meta.accumulated_weight == tx.weight
if tx.hash in self.soft_voided_tx_ids:
voided_by.add(settings.SOFT_VOIDED_ID)
voided_by.add(tx.hash)
if meta.conflict_with:
voided_by.add(tx.hash)
# We must save before marking conflicts as voided because
# the conflicting tx might affect this tx's voided_by metadata.
if voided_by:
meta.voided_by = voided_by.copy()
tx.storage.save_transaction(tx, only_metadata=True)
tx.storage.del_from_indexes(tx)
# Check conflicts of the transactions voiding us.
for h in voided_by:
if h == settings.SOFT_VOIDED_ID:
continue
if h == tx.hash:
continue
tx2 = tx.storage.get_transaction(h)
if not tx2.is_block:
assert isinstance(tx2, Transaction)
self.check_conflicts(tx2)
# Mark voided conflicts as voided.
for h in meta.conflict_with or []:
conflict_tx = cast(Transaction, tx.storage.get_transaction(h))
conflict_tx_meta = conflict_tx.get_metadata()
if conflict_tx_meta.voided_by:
self.mark_as_voided(conflict_tx)
# Finally, check our conflicts.
meta = tx.get_metadata()
if meta.voided_by == {tx.hash}:
self.check_conflicts(tx)
# Assert the final state is valid.
self.assert_valid_consensus(tx)
def check_twins(self, tx: Transaction,
transactions: Iterable[BaseTransaction]) -> None:
""" Check if the tx has any twins in transactions list
A twin tx is a tx that has the same inputs and outputs
We add all the hashes of the twin txs in the metadata
:param transactions: list of transactions to be checked if they are twins with self
"""
assert tx.hash is not None
assert tx.storage is not None
# Getting tx metadata to save the new twins
meta = tx.get_metadata()
# Sorting inputs and outputs for easier validation
sorted_inputs = sorted(tx.inputs,
key=lambda x: (x.tx_id, x.index, x.data))
sorted_outputs = sorted(tx.outputs, key=lambda x: (x.script, x.value))
for candidate in transactions:
assert candidate.hash is not None
# If quantity of inputs is different, it's not a twin.
if len(candidate.inputs) != len(tx.inputs):
continue
# If quantity of outputs is different, it's not a twin.
if len(candidate.outputs) != len(tx.outputs):
continue
# If the hash is the same, it's not a twin.
if candidate.hash == tx.hash:
continue
# Verify if all the inputs are the same
equal = True
for index, tx_input in enumerate(
sorted(candidate.inputs,
key=lambda x: (x.tx_id, x.index, x.data))):
if (tx_input.tx_id != sorted_inputs[index].tx_id
or tx_input.data != sorted_inputs[index].data
or tx_input.index != sorted_inputs[index].index):
equal = False
break
# Verify if all the outputs are the same
if equal:
for index, tx_output in enumerate(
sorted(candidate.outputs,
key=lambda x: (x.script, x.value))):
if (tx_output.value != sorted_outputs[index].value or
tx_output.script != sorted_outputs[index].script):
equal = False
break
# If everything is equal we add in both metadatas
if equal:
meta.twins.append(candidate.hash)
tx_meta = candidate.get_metadata()
tx_meta.twins.append(tx.hash)
tx.storage.save_transaction(candidate, only_metadata=True)
tx.storage.save_transaction(tx, only_metadata=True)
