def update_tx(self, tx: BaseTransaction, *, relax_assert: bool = False) -> None:
""" Update a tx according to its children.
"""
assert tx.storage is not None
assert tx.hash is not None
meta = tx.get_metadata()
if meta.voided_by:
if not relax_assert:
assert tx.hash not in self.tx_last_interval
return
pi = self.tx_last_interval[tx.hash]
min_timestamp = inf
for child_hash in meta.children:
if child_hash in self.tx_last_interval:
child = tx.storage.get_transaction(child_hash)
min_timestamp = min(min_timestamp, child.timestamp)
if min_timestamp != pi.end:
self.tree.remove(pi)
new_interval = Interval(pi.begin, min_timestamp, pi.data)
self.tree.add(new_interval)
self.tx_last_interval[tx.hash] = new_interval
def add_tx(self, tx: BaseTransaction) -> None:
""" Add a new transaction to the index
:param tx: Transaction to be added
"""
assert tx.hash is not None
assert tx.storage is not None
if tx.hash in self.tx_last_interval:
return
# Fix the end of the interval of its parents.
for parent_hash in tx.parents:
pi = self.tx_last_interval.get(parent_hash, None)
if not pi:
continue
if tx.timestamp < pi.end:
self.tree.remove(pi)
new_interval = Interval(pi.begin, tx.timestamp, pi.data)
self.tree.add(new_interval)
self.tx_last_interval[parent_hash] = new_interval
# Check whether any children has already been added.
# It so, the end of the interval is equal to the smallest timestamp of the children.
min_timestamp = inf
meta = tx.get_metadata()
for child_hash in meta.children:
if child_hash in self.tx_last_interval:
child = tx.storage.get_transaction(child_hash)
min_timestamp = min(min_timestamp, child.timestamp)
# Add the interval to the tree.
interval = Interval(tx.timestamp, min_timestamp, tx.hash)
self.tree.add(interval)
self.tx_last_interval[tx.hash] = interval
def assert_valid_consensus(self, tx: BaseTransaction) -> None:
"""Assert the conflict resolution is valid."""
meta = tx.get_metadata()
is_tx_executed = bool(not meta.voided_by)
for h in meta.conflict_with or []:
assert tx.storage is not None
conflict_tx = cast(Transaction, tx.storage.get_transaction(h))
conflict_tx_meta = conflict_tx.get_metadata()
is_conflict_tx_executed = bool(not conflict_tx_meta.voided_by)
assert not (is_tx_executed and is_conflict_tx_executed)
def get_node_label(self, tx: BaseTransaction) -> str:
""" Return the node's label for tx.
"""
assert tx.hash is not None
parts = [tx.hash.hex()[-4:]]
if self.show_weight:
parts.append('w: {:.2f}'.format(tx.weight))
if self.show_acc_weight:
meta = tx.get_metadata()
parts.append('a: {:.2f}'.format(meta.accumulated_weight))
return '\n'.join(parts)
def get_node_attrs(self, tx: BaseTransaction) -> Dict[str, str]:
""" Return node's attributes.
"""
assert tx.hash is not None
node_attrs = {'label': self.get_node_label(tx)}
if tx.is_block:
node_attrs.update(self.block_attrs)
if tx.is_genesis:
node_attrs.update(self.genesis_attrs)
meta = tx.get_metadata()
if meta.voided_by and len(meta.voided_by) > 0:
node_attrs.update(self.voided_attrs)
if meta.voided_by and tx.hash in meta.voided_by:
node_attrs.update(self.conflict_attrs)
return node_attrs
def tx_neighborhood(self,
tx: BaseTransaction,
format: str = 'pdf',
max_level: int = 2,
graph_type: str = 'verification') -> Digraph:
""" Draw the blocks and transactions around `tx`.
:params max_level: Maximum distance between `tx` and the others.
:params graph_type: Graph type to be generated. Possibilities are 'verification' and 'funds'
"""
dot = Digraph(format=format)
dot.attr(rankdir='RL')
dot.attr('node', shape='oval', style='')
root = tx
to_visit = [(0, tx)]
seen = set([tx.hash])
while to_visit:
level, tx = to_visit.pop()
assert tx.hash is not None
assert tx.storage is not None
name = tx.hash.hex()
node_attrs = self.get_node_attrs(tx)
if tx.hash == root.hash:
node_attrs.update(dict(style='filled', penwidth='5.0'))
meta = tx.get_metadata()
if graph_type == 'verification':
if tx.is_block:
continue
dot.node(name, **node_attrs)
if level <= max_level:
for h in chain(tx.parents, meta.children):
if h not in seen:
seen.add(h)
tx2 = tx.storage.get_transaction(h)
to_visit.append((level + 1, tx2))
for h in tx.parents:
if h in seen:
dot.edge(name, h.hex())
elif graph_type == 'funds':
dot.node(name, **node_attrs)
if level <= max_level:
spent_outputs_ids = chain.from_iterable(
meta.spent_outputs.values())
tx_input_ids = [txin.tx_id for txin in tx.inputs]
for h in chain(tx_input_ids, spent_outputs_ids):
if h not in seen:
seen.add(h)
tx2 = tx.storage.get_transaction(h)
to_visit.append((level + 1, tx2))
for txin in tx.inputs:
if txin.tx_id in seen:
dot.edge(name, txin.tx_id.hex())
return dot
def on_new_tx(self,
tx: BaseTransaction,
*,
conn: Optional[HathorProtocol] = None,
quiet: bool = False,
fails_silently: bool = True,
propagate_to_peers: bool = True,
skip_block_weight_verification: bool = False,
sync_checkpoints: bool = False,
partial: bool = False) -> bool:
""" New method for adding transactions or blocks that steps the validation state machine.
:param tx: transaction to be added
:param conn: optionally specify the protocol instance where this tx was received from
:param quiet: if True will not log when a new tx is accepted
:param fails_silently: if False will raise an exception when tx cannot be added
:param propagate_to_peers: if True will relay the tx to other peers if it is accepted
:param skip_block_weight_verification: if True will not check the tx PoW
:param sync_checkpoints: if True and also partial=True, will try to validate as a checkpoint and set the proper
validation state, this is used for adding txs from the sync-checkpoints phase
:param partial: if True will accept txs that can't be fully validated yet (because of missing parent/input) but
will run a basic validation of what can be validated (PoW and other basic fields)
"""
assert tx.hash is not None
if self.tx_storage.transaction_exists(tx.hash):
if not fails_silently:
raise InvalidNewTransaction(
'Transaction already exists {}'.format(tx.hash_hex))
self.log.warn('on_new_tx(): Transaction already exists',
tx=tx.hash_hex)
return False
if tx.timestamp - self.reactor.seconds(
) > settings.MAX_FUTURE_TIMESTAMP_ALLOWED:
if not fails_silently:
raise InvalidNewTransaction(
'Ignoring transaction in the future {} (timestamp={})'.
format(tx.hash_hex, tx.timestamp))
self.log.warn('on_new_tx(): Ignoring transaction in the future',
tx=tx.hash_hex,
future_timestamp=tx.timestamp)
return False
tx.storage = self.tx_storage
try:
metadata = tx.get_metadata()
except TransactionDoesNotExist:
if not fails_silently:
raise InvalidNewTransaction('missing parent')
self.log.warn('on_new_tx(): missing parent', tx=tx.hash_hex)
return False
if metadata.validation.is_invalid():
if not fails_silently:
raise InvalidNewTransaction('previously marked as invalid')
self.log.warn('on_new_tx(): previously marked as invalid',
tx=tx.hash_hex)
return False
# if partial=False (the default) we don't even try to partially validate transactions
if not partial or (metadata.validation.is_fully_connected()
or tx.can_validate_full()):
if isinstance(tx, Transaction) and self.tx_storage.is_tx_needed(
tx.hash):
tx._height_cache = self.tx_storage.needed_index_height(tx.hash)
if not metadata.validation.is_fully_connected():
try:
tx.validate_full(sync_checkpoints=sync_checkpoints)
except HathorError as e:
if not fails_silently:
raise InvalidNewTransaction(
'full validation failed') from e
self.log.warn('on_new_tx(): full validation failed',
tx=tx.hash_hex,
exc_info=True)
return False
# The method below adds the tx as a child of the parents
# This needs to be called right before the save because we were adding the children
# in the tx parents even if the tx was invalid (failing the verifications above)
# then I would have a children that was not in the storage
tx.update_initial_metadata()
self.tx_storage.save_transaction(tx, add_to_indexes=True)
try:
self.consensus_algorithm.update(tx)
except HathorError as e:
if not fails_silently:
raise InvalidNewTransaction(
'consensus update failed') from e
self.log.warn('on_new_tx(): consensus update failed',
tx=tx.hash_hex)
return False
else:
assert tx.validate_full(skip_block_weight_verification=True)
self.tx_fully_validated(tx)
elif sync_checkpoints:
metadata.children = self.tx_storage.children_from_deps(tx.hash)
try:
tx.validate_checkpoint(self.checkpoints)
except HathorError:
if not fails_silently:
raise InvalidNewTransaction('checkpoint validation failed')
self.log.warn('on_new_tx(): checkpoint validation failed',
tx=tx.hash_hex,
exc_info=True)
return False
self.tx_storage.save_transaction(tx)
self.tx_storage.add_to_deps_index(tx.hash,
tx.get_all_dependencies())
self.tx_storage.add_needed_deps(tx)
else:
if isinstance(tx, Block) and not tx.has_basic_block_parent():
if not fails_silently:
raise InvalidNewTransaction(
'block parent needs to be at least basic-valid')
self.log.warn(
'on_new_tx(): block parent needs to be at least basic-valid',
tx=tx.hash_hex)
return False
if not tx.validate_basic():
if not fails_silently:
raise InvalidNewTransaction('basic validation failed')
self.log.warn('on_new_tx(): basic validation failed',
tx=tx.hash_hex)
return False
# The method below adds the tx as a child of the parents
# This needs to be called right before the save because we were adding the children
# in the tx parents even if the tx was invalid (failing the verifications above)
# then I would have a children that was not in the storage
tx.update_initial_metadata()
self.tx_storage.save_transaction(tx)
self.tx_storage.add_to_deps_index(tx.hash,
tx.get_all_dependencies())
self.tx_storage.add_needed_deps(tx)
if tx.is_transaction:
self.tx_storage.remove_from_needed_index(tx.hash)
try:
self.step_validations([tx])
except (AssertionError, HathorError) as e:
if not fails_silently:
raise InvalidNewTransaction('step validations failed') from e
self.log.warn('on_new_tx(): step validations failed',
tx=tx.hash_hex,
exc_info=True)
return False
if not quiet:
ts_date = datetime.datetime.fromtimestamp(tx.timestamp)
now = datetime.datetime.fromtimestamp(self.reactor.seconds())
if tx.is_block:
self.log.info('new block',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now(now))
else:
self.log.info('new tx',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now(now))
if propagate_to_peers:
# Propagate to our peers.
self.connections.send_tx_to_peers(tx)
return True
def _score_block_dfs(self, block: BaseTransaction, used: Set[bytes],
mark_as_best_chain: bool,
newest_timestamp: int) -> float:
""" Internal method to run a DFS. It is used by `calculate_score()`.
"""
assert block.storage is not None
assert block.hash is not None
assert block.is_block
storage = block.storage
from hathor.transaction import Block
score = block.weight
for parent in block.get_parents():
if parent.is_block:
assert isinstance(parent, Block)
if parent.timestamp <= newest_timestamp:
meta = parent.get_metadata()
x = meta.score
else:
x = self._score_block_dfs(parent, used, mark_as_best_chain,
newest_timestamp)
score = sum_weights(score, x)
else:
from hathor.transaction.storage.traversal import BFSWalk
bfs = BFSWalk(storage,
is_dag_verifications=True,
is_left_to_right=False)
for tx in bfs.run(parent, skip_root=False):
assert tx.hash is not None
assert not tx.is_block
if tx.hash in used:
bfs.skip_neighbors(tx)
continue
used.add(tx.hash)
meta = tx.get_metadata()
if meta.first_block:
first_block = storage.get_transaction(meta.first_block)
if first_block.timestamp <= newest_timestamp:
bfs.skip_neighbors(tx)
continue
if mark_as_best_chain:
assert meta.first_block is None
meta.first_block = block.hash
storage.save_transaction(tx, only_metadata=True)
score = sum_weights(score, tx.weight)
# Always save the score when it is calculated.
meta = block.get_metadata()
if not meta.score:
meta.score = score
storage.save_transaction(block, only_metadata=True)
else:
# The score of a block is immutable since the sub-DAG behind it is immutable as well.
# Thus, if we have already calculated it, we just check the consistency of the calculation.
# Unfortunately we may have to calculate it more than once when a new block arrives in a side
# side because the `first_block` points only to the best chain.
assert abs(meta.score - score) < 1e-10, \
'hash={} meta.score={} score={}'.format(block.hash.hex(), meta.score, score)
return score
def on_new_tx(self,
tx: BaseTransaction,
*,
conn: Optional[HathorProtocol] = None,
quiet: bool = False,
fails_silently: bool = True,
propagate_to_peers: bool = True,
skip_block_weight_verification: bool = False) -> bool:
"""This method is called when any transaction arrive.
If `fails_silently` is False, it may raise either InvalidNewTransaction or TxValidationError.
:return: True if the transaction was accepted
:rtype: bool
"""
assert tx.hash is not None
if self.state != self.NodeState.INITIALIZING:
if self.tx_storage.transaction_exists(tx.hash):
if not fails_silently:
raise InvalidNewTransaction(
'Transaction already exists {}'.format(tx.hash_hex))
self.log.debug('on_new_tx(): Transaction already exists',
tx=tx.hash_hex)
return False
if self.state != self.NodeState.INITIALIZING or self._full_verification:
try:
assert self.validate_new_tx(
tx,
skip_block_weight_verification=
skip_block_weight_verification) is True
except (InvalidNewTransaction, TxValidationError):
# Discard invalid Transaction/block.
self.log.debug('tx/block discarded', tx=tx, exc_info=True)
if not fails_silently:
raise
return False
if self.state != self.NodeState.INITIALIZING:
self.tx_storage.save_transaction(tx)
else:
self.tx_storage._add_to_cache(tx)
if self._full_verification:
tx.reset_metadata()
else:
# When doing a fast init, we don't update the consensus, so we must trust the data on the metadata
# For transactions, we don't store them on the tips index if they are voided
# We have to execute _add_to_cache before because _del_from_cache does not remove from all indexes
metadata = tx.get_metadata()
if not tx.is_block and metadata.voided_by:
self.tx_storage._del_from_cache(tx)
if self.state != self.NodeState.INITIALIZING or self._full_verification:
try:
tx.update_initial_metadata()
self.consensus_algorithm.update(tx)
except Exception:
self.log.exception('unexpected error when processing tx',
tx=tx)
self.tx_storage.remove_transaction(tx)
raise
if not quiet:
ts_date = datetime.datetime.fromtimestamp(tx.timestamp)
if tx.is_block:
self.log.info('new block',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now())
else:
self.log.info('new tx',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now())
if propagate_to_peers:
# Propagate to our peers.
self.connections.send_tx_to_peers(tx)
if self.wallet:
# TODO Remove it and use pubsub instead.
self.wallet.on_new_tx(tx)
# Publish to pubsub manager the new tx accepted
self.pubsub.publish(HathorEvents.NETWORK_NEW_TX_ACCEPTED, tx=tx)
return True
def update(self, tx: BaseTransaction) -> None:
assert tx.hash is not None
assert tx.storage is not None
to_remove: Set[bytes] = set()
to_remove_parents: Set[bytes] = set()
tx_storage = tx.storage
for tip_tx in self.iter(tx_storage):
assert tip_tx.hash is not None
meta = tip_tx.get_metadata()
# a new tx/block added might cause a tx in the tips to become voided. For instance, there might be a tx1 a
# double spending tx2, where tx1 is valid and tx2 voided. A new block confirming tx2 will make it valid
# while tx1 becomes voided, so it has to be removed from the tips. The txs confirmed by tx1 need to be
# double checked, as they might themselves become tips (hence we use to_remove_parents)
if meta.voided_by:
to_remove.add(tip_tx.hash)
to_remove_parents.update(tip_tx.parents)
continue
# might also happen that a tip has a child that became valid, so it's not a tip anymore
confirmed = False
for child_meta in map(tx_storage.get_metadata, meta.children):
assert child_meta is not None
if not child_meta.voided_by:
confirmed = True
break
if confirmed:
to_remove.add(tip_tx.hash)
if to_remove:
self._discard_many(to_remove)
self.log.debug('removed voided txs from tips', txs=[tx.hex() for tx in to_remove])
# Check if any of the txs being confirmed by the voided txs is a tip again. This happens
# if it doesn't have any other valid child.
to_add = set()
for tx_hash in to_remove_parents:
confirmed = False
# check if it has any valid children
meta = not_none(tx_storage.get_metadata(tx_hash))
if meta.voided_by:
continue
children = meta.children
for child_meta in map(tx_storage.get_metadata, children):
assert child_meta is not None
if not child_meta.voided_by:
confirmed = True
break
if not confirmed:
to_add.add(tx_hash)
if to_add:
self._add_many(to_add)
self.log.debug('added txs to tips', txs=[tx.hex() for tx in to_add])
if tx.get_metadata().voided_by:
# this tx is voided, don't need to update the tips
self.log.debug('voided tx, won\'t add it as a tip', tx=tx.hash_hex)
return
self._discard_many(set(tx.parents))
if tx.is_transaction and tx.get_metadata().first_block is None:
assert tx.hash is not None
self._add(tx.hash)