def validate_new_tx(self,
tx: BaseTransaction,
skip_block_weight_verification: bool = False) -> bool:
""" Process incoming transaction during initialization.
These transactions came only from storage.
"""
assert tx.hash is not None
if self.state == self.NodeState.INITIALIZING:
if tx.is_genesis:
return True
else:
if tx.is_genesis:
raise InvalidNewTransaction('Genesis? {}'.format(tx.hash_hex))
if tx.timestamp - self.reactor.seconds(
) > settings.MAX_FUTURE_TIMESTAMP_ALLOWED:
raise InvalidNewTransaction(
'Ignoring transaction in the future {} (timestamp={})'.format(
tx.hash_hex, tx.timestamp))
# Verify transaction and raises an TxValidationError if tx is not valid.
tx.verify()
if tx.is_block:
tx = cast(Block, tx)
assert tx.hash is not None # XXX: it appears that after casting this assert "casting" is lost
if not skip_block_weight_verification:
# Validate minimum block difficulty
block_weight = self.calculate_block_difficulty(tx)
if tx.weight < block_weight - settings.WEIGHT_TOL:
raise InvalidNewTransaction(
'Invalid new block {}: weight ({}) is smaller than the minimum weight ({})'
.format(tx.hash.hex(), tx.weight, block_weight))
parent_block = tx.get_block_parent()
tokens_issued_per_block = self.get_tokens_issued_per_block(
parent_block.get_metadata().height + 1)
if tx.sum_outputs != tokens_issued_per_block:
raise InvalidNewTransaction(
'Invalid number of issued tokens tag=invalid_issued_tokens'
' tx.hash={tx.hash_hex} issued={tx.sum_outputs} allowed={allowed}'
.format(
tx=tx,
allowed=tokens_issued_per_block,
))
else:
assert tx.hash is not None # XXX: it appears that after casting this assert "casting" is lost
# Validate minimum tx difficulty
min_tx_weight = self.minimum_tx_weight(tx)
if tx.weight < min_tx_weight - settings.WEIGHT_TOL:
raise InvalidNewTransaction(
'Invalid new tx {}: weight ({}) is smaller than the minimum weight ({})'
.format(tx.hash_hex, tx.weight, min_tx_weight))
return True
def add_tx(self, tx: BaseTransaction) -> None:
# if it's a TokenCreationTransaction, update name and symbol
self.log.debug('add_tx', tx=tx.hash_hex, ver=tx.version)
if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION:
from hathor.transaction.token_creation_tx import TokenCreationTransaction
tx = cast(TokenCreationTransaction, tx)
assert tx.hash is not None
self.log.debug('create_token_info',
tx=tx.hash_hex,
name=tx.token_name,
symb=tx.token_symbol)
self._create_token_info(tx.hash, tx.token_name, tx.token_symbol)
if tx.is_transaction:
# Adding this tx to the transactions key list
assert isinstance(tx, Transaction)
for token_uid in tx.tokens:
assert tx.hash is not None
self._add_transaction(token_uid, tx.timestamp, tx.hash)
for tx_input in tx.inputs:
spent_tx = tx.get_spent_tx(tx_input)
self._remove_utxo(spent_tx, tx_input.index)
for index in range(len(tx.outputs)):
self.log.debug('add utxo', tx=tx.hash_hex, index=index)
self._add_utxo(tx, index)
def publish_tx(self,
tx: BaseTransaction,
*,
addresses: Optional[Iterable[str]] = None) -> None:
""" Publish WALLET_ADDRESS_HISTORY for all addresses of a transaction.
"""
from hathor.pubsub import HathorEvents
if not self.pubsub:
return
if addresses is None:
addresses = tx.get_related_addresses()
data = tx.to_json_extended()
for address in addresses:
self.pubsub.publish(HathorEvents.WALLET_ADDRESS_HISTORY,
address=address,
history=data)
def minimum_tx_weight(self, tx: BaseTransaction) -> float:
""" Returns the minimum weight for the param tx
The minimum is calculated by the following function:
w = alpha * log(size, 2) + 4.0 + 4.0
----------------
1 + k / amount
:param tx: tx to calculate the minimum weight
:type tx: :py:class:`hathor.transaction.transaction.Transaction`
:return: minimum weight for the tx
:rtype: float
"""
# In test mode we don't validate the minimum weight for tx
# We do this to allow generating many txs for testing
if self.test_mode & TestMode.TEST_TX_WEIGHT:
return 1
if tx.is_genesis:
return self.min_tx_weight
tx_size = len(tx.get_struct())
# We need to take into consideration the decimal places because it is inside the amount.
# For instance, if one wants to transfer 20 HTRs, the amount will be 2000.
# Max below is preventing division by 0 when handling authority methods that have no outputs
amount = max(1, tx.sum_outputs) / (10**settings.DECIMAL_PLACES)
weight = (+self.min_tx_weight_coefficient * log(tx_size, 2) + 4 /
(1 + self.min_tx_weight_k / amount) + 4)
# Make sure the calculated weight is at least the minimum
weight = max(weight, self.min_tx_weight)
return weight
def add_tx(self, tx: BaseTransaction) -> None:
""" Add a new transaction to the index
:param tx: Transaction to be added
"""
assert tx.hash is not None
assert tx.storage is not None
if tx.hash in self.tx_last_interval:
return
# Fix the end of the interval of its parents.
for parent_hash in tx.parents:
pi = self.tx_last_interval.get(parent_hash, None)
if not pi:
continue
if tx.timestamp < pi.end:
self.tree.remove(pi)
new_interval = Interval(pi.begin, tx.timestamp, pi.data)
self.tree.add(new_interval)
self.tx_last_interval[parent_hash] = new_interval
# Check whether any children has already been added.
# It so, the end of the interval is equal to the smallest timestamp of the children.
min_timestamp = inf
meta = tx.get_metadata()
for child_hash in meta.children:
if child_hash in self.tx_last_interval:
child = tx.storage.get_transaction(child_hash)
min_timestamp = min(min_timestamp, child.timestamp)
# Add the interval to the tree.
interval = Interval(tx.timestamp, min_timestamp, tx.hash)
self.tree.add(interval)
self.tx_last_interval[tx.hash] = interval
def update_tx(self, tx: BaseTransaction, *, relax_assert: bool = False) -> None:
""" Update a tx according to its children.
"""
assert tx.storage is not None
assert tx.hash is not None
meta = tx.get_metadata()
if meta.voided_by:
if not relax_assert:
assert tx.hash not in self.tx_last_interval
return
pi = self.tx_last_interval[tx.hash]
min_timestamp = inf
for child_hash in meta.children:
if child_hash in self.tx_last_interval:
child = tx.storage.get_transaction(child_hash)
min_timestamp = min(min_timestamp, child.timestamp)
if min_timestamp != pi.end:
self.tree.remove(pi)
new_interval = Interval(pi.begin, min_timestamp, pi.data)
self.tree.add(new_interval)
self.tx_last_interval[tx.hash] = new_interval
def add_tx(self, tx: BaseTransaction) -> None:
""" Checks if this tx has mint or melt inputs/outputs and adds to tokens index
"""
for tx_input in tx.inputs:
spent_tx = tx.get_spent_tx(tx_input)
self._remove_from_index(spent_tx, tx_input.index)
for index in range(len(tx.outputs)):
self._add_to_index(tx, index)
# if it's a TokenCreationTransaction, update name and symbol
if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION:
from hathor.transaction.token_creation_tx import TokenCreationTransaction
tx = cast(TokenCreationTransaction, tx)
assert tx.hash is not None
status = self.tokens[tx.hash]
status.name = tx.token_name
status.symbol = tx.token_symbol
if tx.is_transaction:
# Adding this tx to the transactions key list
assert isinstance(tx, Transaction)
for token_uid in tx.tokens:
transactions = self.tokens[token_uid].transactions
# It is safe to use the in operator because it is O(log(n)).
# http://www.grantjenks.com/docs/sortedcontainers/sortedlist.html#sortedcontainers.SortedList.__contains__
assert tx.hash is not None
element = TransactionIndexElement(tx.timestamp, tx.hash)
if element in transactions:
return
transactions.add(element)
def remove_tx(self, tx: BaseTransaction) -> None:
""" Remove tx inputs and outputs from the wallet index (indexed by its addresses).
"""
assert tx.hash is not None
addresses = tx.get_related_addresses()
for address in addresses:
self.index[address].discard(tx.hash)
def remove_tx(self, tx: BaseTransaction) -> None:
""" Remove tx inputs and outputs from the wallet index (indexed by its addresses).
"""
assert tx.hash is not None
addresses = tx.get_related_addresses()
for address in addresses:
self.log.debug('delete address', address=address)
self._db.delete((self._cf, self._to_key(address, tx)))
def assert_valid_consensus(self, tx: BaseTransaction) -> None:
"""Assert the conflict resolution is valid."""
meta = tx.get_metadata()
is_tx_executed = bool(not meta.voided_by)
for h in meta.conflict_with or []:
assert tx.storage is not None
conflict_tx = cast(Transaction, tx.storage.get_transaction(h))
conflict_tx_meta = conflict_tx.get_metadata()
is_conflict_tx_executed = bool(not conflict_tx_meta.voided_by)
assert not (is_tx_executed and is_conflict_tx_executed)
def add_tx(self, tx: BaseTransaction) -> None:
""" Add tx inputs and outputs to the wallet index (indexed by its addresses).
"""
assert tx.hash is not None
addresses = tx.get_related_addresses()
for address in addresses:
self.index[address].add(tx.hash)
self.publish_tx(tx, addresses=addresses)
def get_node_label(self, tx: BaseTransaction) -> str:
""" Return the node's label for tx.
"""
assert tx.hash is not None
parts = [tx.hash.hex()[-4:]]
if self.show_weight:
parts.append('w: {:.2f}'.format(tx.weight))
if self.show_acc_weight:
meta = tx.get_metadata()
parts.append('a: {:.2f}'.format(meta.accumulated_weight))
return '\n'.join(parts)
def add_tx(self, tx: BaseTransaction) -> None:
""" Add tx inputs and outputs to the wallet index (indexed by its addresses).
"""
assert tx.hash is not None
addresses = tx.get_related_addresses()
for address in addresses:
self.log.debug('put address', address=address)
self._db.put((self._cf, self._to_key(address, tx)), b'')
self.publish_tx(tx, addresses=addresses)
def propagate_tx(self,
tx: BaseTransaction,
fails_silently: bool = True) -> bool:
"""Push a new transaction to the network. It is used by both the wallet and the mining modules.
:return: True if the transaction was accepted
:rtype: bool
"""
if tx.storage:
assert tx.storage == self.tx_storage, 'Invalid tx storage'
else:
tx.storage = self.tx_storage
return self.on_new_tx(tx, fails_silently=fails_silently)
def _remove_from_index(self, tx: BaseTransaction, index: int) -> None:
""" Remove tx from mint/melt indexes and total amount
"""
assert tx.hash is not None
tx_output = tx.outputs[index]
token_uid = tx.get_token_uid(tx_output.get_token_index())
if tx_output.is_token_authority():
if tx_output.can_mint_token():
# remove from mint index
self.tokens[token_uid].mint.discard((tx.hash, index))
if tx_output.can_melt_token():
# remove from melt index
self.tokens[token_uid].melt.discard((tx.hash, index))
else:
self.tokens[token_uid].total -= tx_output.value
def _add_to_index(self, tx: BaseTransaction, index: int) -> None:
""" Add tx to mint/melt indexes and total amount
"""
assert tx.hash is not None
tx_output = tx.outputs[index]
token_uid = tx.get_token_uid(tx_output.get_token_index())
if tx_output.is_token_authority():
if tx_output.can_mint_token():
# add to mint index
self.tokens[token_uid].mint.add((tx.hash, index))
if tx_output.can_melt_token():
# add to melt index
self.tokens[token_uid].melt.add((tx.hash, index))
else:
self.tokens[token_uid].total += tx_output.value
def get_node_attrs(self, tx: BaseTransaction) -> Dict[str, str]:
""" Return node's attributes.
"""
assert tx.hash is not None
node_attrs = {'label': self.get_node_label(tx)}
if tx.is_block:
node_attrs.update(self.block_attrs)
if tx.is_genesis:
node_attrs.update(self.genesis_attrs)
meta = tx.get_metadata()
if meta.voided_by and len(meta.voided_by) > 0:
node_attrs.update(self.voided_attrs)
if meta.voided_by and tx.hash in meta.voided_by:
node_attrs.update(self.conflict_attrs)
return node_attrs
def del_tx(self, tx: BaseTransaction) -> None:
for tx_input in tx.inputs:
spent_tx = tx.get_spent_tx(tx_input)
self._add_utxo(spent_tx, tx_input.index)
for index in range(len(tx.outputs)):
self._remove_utxo(tx, index)
if tx.is_transaction:
# Removing this tx from the transactions key list
assert isinstance(tx, Transaction)
for token_uid in tx.tokens:
assert tx.hash is not None
self._remove_transaction(token_uid, tx.timestamp, tx.hash)
# if it's a TokenCreationTransaction, remove it from index
if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION:
assert tx.hash is not None
self._destroy_token(tx.hash)
def del_tx(self, tx: BaseTransaction) -> None:
for tx_input in tx.inputs:
spent_tx = tx.get_spent_tx(tx_input)
self._add_to_index(spent_tx, tx_input.index)
for index in range(len(tx.outputs)):
self._remove_from_index(tx, index)
if tx.is_transaction:
# Removing this tx from the transactions key list
assert isinstance(tx, Transaction)
for token_uid in tx.tokens:
transactions = self._tokens[token_uid]._transactions
idx = transactions.bisect_key_left((tx.timestamp, tx.hash))
if idx < len(transactions) and transactions[idx].hash == tx.hash:
transactions.pop(idx)
# if it's a TokenCreationTransaction, remove it from index
if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION:
assert tx.hash is not None
del self._tokens[tx.hash]
def _remove_utxo(self, tx: BaseTransaction, index: int) -> None:
""" Remove tx from mint/melt indexes and total amount
"""
assert tx.hash is not None
tx_output = tx.outputs[index]
token_uid = tx.get_token_uid(tx_output.get_token_index())
if tx_output.is_token_authority():
if tx_output.can_mint_token():
# remove from mint index
self._remove_authority_utxo(token_uid,
tx.hash,
index,
is_mint=True)
if tx_output.can_melt_token():
# remove from melt index
self._remove_authority_utxo(token_uid,
tx.hash,
index,
is_mint=False)
else:
self._subtract_from_total(token_uid, tx_output.value)
def _add_utxo(self, tx: BaseTransaction, index: int) -> None:
""" Add tx to mint/melt indexes and total amount
"""
assert tx.hash is not None
tx_output = tx.outputs[index]
token_uid = tx.get_token_uid(tx_output.get_token_index())
if tx_output.is_token_authority():
if tx_output.can_mint_token():
# add to mint index
self._add_authority_utxo(token_uid,
tx.hash,
index,
is_mint=True)
if tx_output.can_melt_token():
# add to melt index
self._add_authority_utxo(token_uid,
tx.hash,
index,
is_mint=False)
else:
self._add_to_total(token_uid, tx_output.value)
def on_new_tx(self, tx: BaseTransaction) -> None:
"""Checks the inputs and outputs of a transaction for matching keys.
If an output matches, will add it to the unspent_txs dict.
If an input matches, removes from unspent_txs dict and adds to spent_txs dict.
"""
assert tx.hash is not None
updated = False
# check outputs
for index, output in enumerate(tx.outputs):
script_type_out = parse_address_script(output.script)
if script_type_out:
if script_type_out.address in self.keys:
token_id = tx.get_token_uid(output.get_token_index())
# this wallet received tokens
utxo = UnspentTx(tx.hash,
index,
output.value,
tx.timestamp,
script_type_out.address,
output.token_data,
timelock=script_type_out.timelock)
self.unspent_txs[token_id][(tx.hash, index)] = utxo
# mark key as used
self.tokens_received(script_type_out.address)
updated = True
# publish new output and new balance
self.publish_update(HathorEvents.WALLET_OUTPUT_RECEIVED,
total=self.get_total_tx(),
output=utxo)
else:
# it's the only one we know, so log warning
self.log.warn('unknown script')
# check inputs
for _input in tx.inputs:
assert tx.storage is not None
output_tx = tx.storage.get_transaction(_input.tx_id)
output = output_tx.outputs[_input.index]
token_id = output_tx.get_token_uid(output.get_token_index())
script_type_out = parse_address_script(output.script)
if script_type_out:
if script_type_out.address in self.keys:
# this wallet spent tokens
# remove from unspent_txs
key = (_input.tx_id, _input.index)
old_utxo = self.unspent_txs[token_id].pop(key, None)
if old_utxo is None:
old_utxo = self.maybe_spent_txs[token_id].pop(
key, None)
if old_utxo:
# add to spent_txs
spent = SpentTx(tx.hash, _input.tx_id, _input.index,
old_utxo.value, tx.timestamp)
self.spent_txs[key].append(spent)
updated = True
# publish spent output and new balance
self.publish_update(HathorEvents.WALLET_INPUT_SPENT,
output_spent=spent)
else:
# If we dont have it in the unspent_txs, it must be in the spent_txs
# So we append this spent with the others
if key in self.spent_txs:
output_tx = tx.storage.get_transaction(
_input.tx_id)
output = output_tx.outputs[_input.index]
spent = SpentTx(tx.hash, _input.tx_id,
_input.index, output.value,
tx.timestamp)
self.spent_txs[key].append(spent)
else:
self.log.warn('unknown input data')
if updated:
# TODO update history file?
# XXX should wallet always update it or it will be called externally?
self.update_balance()
def on_new_tx(self,
tx: BaseTransaction,
*,
conn: Optional[HathorProtocol] = None,
quiet: bool = False,
fails_silently: bool = True,
propagate_to_peers: bool = True,
skip_block_weight_verification: bool = False) -> bool:
"""This method is called when any transaction arrive.
If `fails_silently` is False, it may raise either InvalidNewTransaction or TxValidationError.
:return: True if the transaction was accepted
:rtype: bool
"""
assert tx.hash is not None
if self.state != self.NodeState.INITIALIZING:
if self.tx_storage.transaction_exists(tx.hash):
if not fails_silently:
raise InvalidNewTransaction(
'Transaction already exists {}'.format(tx.hash_hex))
self.log.debug('on_new_tx(): Transaction already exists',
tx=tx.hash_hex)
return False
if self.state != self.NodeState.INITIALIZING or self._full_verification:
try:
assert self.validate_new_tx(
tx,
skip_block_weight_verification=
skip_block_weight_verification) is True
except (InvalidNewTransaction, TxValidationError):
# Discard invalid Transaction/block.
self.log.debug('tx/block discarded', tx=tx, exc_info=True)
if not fails_silently:
raise
return False
if self.state != self.NodeState.INITIALIZING:
self.tx_storage.save_transaction(tx)
else:
self.tx_storage._add_to_cache(tx)
if self._full_verification:
tx.reset_metadata()
else:
# When doing a fast init, we don't update the consensus, so we must trust the data on the metadata
# For transactions, we don't store them on the tips index if they are voided
# We have to execute _add_to_cache before because _del_from_cache does not remove from all indexes
metadata = tx.get_metadata()
if not tx.is_block and metadata.voided_by:
self.tx_storage._del_from_cache(tx)
if self.state != self.NodeState.INITIALIZING or self._full_verification:
try:
tx.update_initial_metadata()
self.consensus_algorithm.update(tx)
except Exception:
self.log.exception('unexpected error when processing tx',
tx=tx)
self.tx_storage.remove_transaction(tx)
raise
if not quiet:
ts_date = datetime.datetime.fromtimestamp(tx.timestamp)
if tx.is_block:
self.log.info('new block',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now())
else:
self.log.info('new tx',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now())
if propagate_to_peers:
# Propagate to our peers.
self.connections.send_tx_to_peers(tx)
if self.wallet:
# TODO Remove it and use pubsub instead.
self.wallet.on_new_tx(tx)
# Publish to pubsub manager the new tx accepted
self.pubsub.publish(HathorEvents.NETWORK_NEW_TX_ACCEPTED, tx=tx)
return True
def tx_neighborhood(self,
tx: BaseTransaction,
format: str = 'pdf',
max_level: int = 2,
graph_type: str = 'verification') -> Digraph:
""" Draw the blocks and transactions around `tx`.
:params max_level: Maximum distance between `tx` and the others.
:params graph_type: Graph type to be generated. Possibilities are 'verification' and 'funds'
"""
dot = Digraph(format=format)
dot.attr(rankdir='RL')
dot.attr('node', shape='oval', style='')
root = tx
to_visit = [(0, tx)]
seen = set([tx.hash])
while to_visit:
level, tx = to_visit.pop()
assert tx.hash is not None
assert tx.storage is not None
name = tx.hash.hex()
node_attrs = self.get_node_attrs(tx)
if tx.hash == root.hash:
node_attrs.update(dict(style='filled', penwidth='5.0'))
meta = tx.get_metadata()
if graph_type == 'verification':
if tx.is_block:
continue
dot.node(name, **node_attrs)
if level <= max_level:
for h in chain(tx.parents, meta.children):
if h not in seen:
seen.add(h)
tx2 = tx.storage.get_transaction(h)
to_visit.append((level + 1, tx2))
for h in tx.parents:
if h in seen:
dot.edge(name, h.hex())
elif graph_type == 'funds':
dot.node(name, **node_attrs)
if level <= max_level:
spent_outputs_ids = chain.from_iterable(
meta.spent_outputs.values())
tx_input_ids = [txin.tx_id for txin in tx.inputs]
for h in chain(tx_input_ids, spent_outputs_ids):
if h not in seen:
seen.add(h)
tx2 = tx.storage.get_transaction(h)
to_visit.append((level + 1, tx2))
for txin in tx.inputs:
if txin.tx_id in seen:
dot.edge(name, txin.tx_id.hex())
return dot
def _clone(self, x: BaseTransaction) -> BaseTransaction:
if self._clone_if_needed:
return x.clone()
else:
return x
def on_new_tx(self,
tx: BaseTransaction,
*,
conn: Optional[HathorProtocol] = None,
quiet: bool = False,
fails_silently: bool = True,
propagate_to_peers: bool = True) -> bool:
"""This method is called when any transaction arrive.
If `fails_silently` is False, it may raise either InvalidNewTransaction or TxValidationError.
:return: True if the transaction was accepted
:rtype: bool
"""
assert tx.hash is not None
if self.state != self.NodeState.INITIALIZING:
if self.tx_storage.transaction_exists(tx.hash):
if not fails_silently:
raise InvalidNewTransaction(
'Transaction already exists {}'.format(tx.hash.hex()))
self.log.debug(
'on_new_tx(): Already have transaction {}'.format(
tx.hash.hex()))
return False
try:
assert self.validate_new_tx(tx) is True
except (InvalidNewTransaction, TxValidationError) as e:
# Discard invalid Transaction/block.
self.log.debug('Transaction/Block discarded', tx=tx, exc=e)
if not fails_silently:
raise
return False
if self.state != self.NodeState.INITIALIZING:
self.tx_storage.save_transaction(tx)
else:
tx.reset_metadata()
self.tx_storage._add_to_cache(tx)
try:
tx.update_initial_metadata()
self.consensus_algorithm.update(tx)
except Exception:
pretty_json = json.dumps(tx.to_json(), indent=4)
self.log.error(
'An unexpected error occurred when processing {tx.hash_hex}\n'
'{pretty_json}',
tx=tx,
pretty_json=pretty_json)
self.tx_storage.remove_transaction(tx)
raise
if not quiet:
ts_date = datetime.datetime.fromtimestamp(tx.timestamp)
if tx.is_block:
self.log.info('New block found',
tag='new_block',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now())
else:
self.log.info('New transaction found',
tag='new_tx',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now())
if propagate_to_peers:
# Propagate to our peers.
self.connections.send_tx_to_peers(tx)
if self.wallet:
# TODO Remove it and use pubsub instead.
self.wallet.on_new_tx(tx)
# Publish to pubsub manager the new tx accepted
self.pubsub.publish(HathorEvents.NETWORK_NEW_TX_ACCEPTED, tx=tx)
return True
def _score_block_dfs(self, block: BaseTransaction, used: Set[bytes],
mark_as_best_chain: bool,
newest_timestamp: int) -> float:
""" Internal method to run a DFS. It is used by `calculate_score()`.
"""
assert block.storage is not None
assert block.hash is not None
assert block.is_block
storage = block.storage
from hathor.transaction import Block
score = block.weight
for parent in block.get_parents():
if parent.is_block:
assert isinstance(parent, Block)
if parent.timestamp <= newest_timestamp:
meta = parent.get_metadata()
x = meta.score
else:
x = self._score_block_dfs(parent, used, mark_as_best_chain,
newest_timestamp)
score = sum_weights(score, x)
else:
from hathor.transaction.storage.traversal import BFSWalk
bfs = BFSWalk(storage,
is_dag_verifications=True,
is_left_to_right=False)
for tx in bfs.run(parent, skip_root=False):
assert tx.hash is not None
assert not tx.is_block
if tx.hash in used:
bfs.skip_neighbors(tx)
continue
used.add(tx.hash)
meta = tx.get_metadata()
if meta.first_block:
first_block = storage.get_transaction(meta.first_block)
if first_block.timestamp <= newest_timestamp:
bfs.skip_neighbors(tx)
continue
if mark_as_best_chain:
assert meta.first_block is None
meta.first_block = block.hash
storage.save_transaction(tx, only_metadata=True)
score = sum_weights(score, tx.weight)
# Always save the score when it is calculated.
meta = block.get_metadata()
if not meta.score:
meta.score = score
storage.save_transaction(block, only_metadata=True)
else:
# The score of a block is immutable since the sub-DAG behind it is immutable as well.
# Thus, if we have already calculated it, we just check the consistency of the calculation.
# Unfortunately we may have to calculate it more than once when a new block arrives in a side
# side because the `first_block` points only to the best chain.
assert abs(meta.score - score) < 1e-10, \
'hash={} meta.score={} score={}'.format(block.hash.hex(), meta.score, score)
return score
def on_new_tx(self,
tx: BaseTransaction,
*,
conn: Optional[HathorProtocol] = None,
quiet: bool = False,
fails_silently: bool = True,
propagate_to_peers: bool = True,
skip_block_weight_verification: bool = False,
sync_checkpoints: bool = False,
partial: bool = False) -> bool:
""" New method for adding transactions or blocks that steps the validation state machine.
:param tx: transaction to be added
:param conn: optionally specify the protocol instance where this tx was received from
:param quiet: if True will not log when a new tx is accepted
:param fails_silently: if False will raise an exception when tx cannot be added
:param propagate_to_peers: if True will relay the tx to other peers if it is accepted
:param skip_block_weight_verification: if True will not check the tx PoW
:param sync_checkpoints: if True and also partial=True, will try to validate as a checkpoint and set the proper
validation state, this is used for adding txs from the sync-checkpoints phase
:param partial: if True will accept txs that can't be fully validated yet (because of missing parent/input) but
will run a basic validation of what can be validated (PoW and other basic fields)
"""
assert tx.hash is not None
if self.tx_storage.transaction_exists(tx.hash):
if not fails_silently:
raise InvalidNewTransaction(
'Transaction already exists {}'.format(tx.hash_hex))
self.log.warn('on_new_tx(): Transaction already exists',
tx=tx.hash_hex)
return False
if tx.timestamp - self.reactor.seconds(
) > settings.MAX_FUTURE_TIMESTAMP_ALLOWED:
if not fails_silently:
raise InvalidNewTransaction(
'Ignoring transaction in the future {} (timestamp={})'.
format(tx.hash_hex, tx.timestamp))
self.log.warn('on_new_tx(): Ignoring transaction in the future',
tx=tx.hash_hex,
future_timestamp=tx.timestamp)
return False
tx.storage = self.tx_storage
try:
metadata = tx.get_metadata()
except TransactionDoesNotExist:
if not fails_silently:
raise InvalidNewTransaction('missing parent')
self.log.warn('on_new_tx(): missing parent', tx=tx.hash_hex)
return False
if metadata.validation.is_invalid():
if not fails_silently:
raise InvalidNewTransaction('previously marked as invalid')
self.log.warn('on_new_tx(): previously marked as invalid',
tx=tx.hash_hex)
return False
# if partial=False (the default) we don't even try to partially validate transactions
if not partial or (metadata.validation.is_fully_connected()
or tx.can_validate_full()):
if isinstance(tx, Transaction) and self.tx_storage.is_tx_needed(
tx.hash):
tx._height_cache = self.tx_storage.needed_index_height(tx.hash)
if not metadata.validation.is_fully_connected():
try:
tx.validate_full(sync_checkpoints=sync_checkpoints)
except HathorError as e:
if not fails_silently:
raise InvalidNewTransaction(
'full validation failed') from e
self.log.warn('on_new_tx(): full validation failed',
tx=tx.hash_hex,
exc_info=True)
return False
# The method below adds the tx as a child of the parents
# This needs to be called right before the save because we were adding the children
# in the tx parents even if the tx was invalid (failing the verifications above)
# then I would have a children that was not in the storage
tx.update_initial_metadata()
self.tx_storage.save_transaction(tx, add_to_indexes=True)
try:
self.consensus_algorithm.update(tx)
except HathorError as e:
if not fails_silently:
raise InvalidNewTransaction(
'consensus update failed') from e
self.log.warn('on_new_tx(): consensus update failed',
tx=tx.hash_hex)
return False
else:
assert tx.validate_full(skip_block_weight_verification=True)
self.tx_fully_validated(tx)
elif sync_checkpoints:
metadata.children = self.tx_storage.children_from_deps(tx.hash)
try:
tx.validate_checkpoint(self.checkpoints)
except HathorError:
if not fails_silently:
raise InvalidNewTransaction('checkpoint validation failed')
self.log.warn('on_new_tx(): checkpoint validation failed',
tx=tx.hash_hex,
exc_info=True)
return False
self.tx_storage.save_transaction(tx)
self.tx_storage.add_to_deps_index(tx.hash,
tx.get_all_dependencies())
self.tx_storage.add_needed_deps(tx)
else:
if isinstance(tx, Block) and not tx.has_basic_block_parent():
if not fails_silently:
raise InvalidNewTransaction(
'block parent needs to be at least basic-valid')
self.log.warn(
'on_new_tx(): block parent needs to be at least basic-valid',
tx=tx.hash_hex)
return False
if not tx.validate_basic():
if not fails_silently:
raise InvalidNewTransaction('basic validation failed')
self.log.warn('on_new_tx(): basic validation failed',
tx=tx.hash_hex)
return False
# The method below adds the tx as a child of the parents
# This needs to be called right before the save because we were adding the children
# in the tx parents even if the tx was invalid (failing the verifications above)
# then I would have a children that was not in the storage
tx.update_initial_metadata()
self.tx_storage.save_transaction(tx)
self.tx_storage.add_to_deps_index(tx.hash,
tx.get_all_dependencies())
self.tx_storage.add_needed_deps(tx)
if tx.is_transaction:
self.tx_storage.remove_from_needed_index(tx.hash)
try:
self.step_validations([tx])
except (AssertionError, HathorError) as e:
if not fails_silently:
raise InvalidNewTransaction('step validations failed') from e
self.log.warn('on_new_tx(): step validations failed',
tx=tx.hash_hex,
exc_info=True)
return False
if not quiet:
ts_date = datetime.datetime.fromtimestamp(tx.timestamp)
now = datetime.datetime.fromtimestamp(self.reactor.seconds())
if tx.is_block:
self.log.info('new block',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now(now))
else:
self.log.info('new tx',
tx=tx,
ts_date=ts_date,
time_from_now=tx.get_time_from_now(now))
if propagate_to_peers:
# Propagate to our peers.
self.connections.send_tx_to_peers(tx)
return True
def send_data(self, tx: BaseTransaction) -> None:
""" Send a DATA message.
"""
# self.log.debug('Sending {tx.hash_hex}...', tx=tx)
payload = base64.b64encode(tx.get_struct()).decode('ascii')
self.send_message(ProtocolMessages.DATA, payload)
def update(self, tx: BaseTransaction) -> None:
assert tx.hash is not None
assert tx.storage is not None
to_remove: Set[bytes] = set()
to_remove_parents: Set[bytes] = set()
tx_storage = tx.storage
for tip_tx in self.iter(tx_storage):
assert tip_tx.hash is not None
meta = tip_tx.get_metadata()
# a new tx/block added might cause a tx in the tips to become voided. For instance, there might be a tx1 a
# double spending tx2, where tx1 is valid and tx2 voided. A new block confirming tx2 will make it valid
# while tx1 becomes voided, so it has to be removed from the tips. The txs confirmed by tx1 need to be
# double checked, as they might themselves become tips (hence we use to_remove_parents)
if meta.voided_by:
to_remove.add(tip_tx.hash)
to_remove_parents.update(tip_tx.parents)
continue
# might also happen that a tip has a child that became valid, so it's not a tip anymore
confirmed = False
for child_meta in map(tx_storage.get_metadata, meta.children):
assert child_meta is not None
if not child_meta.voided_by:
confirmed = True
break
if confirmed:
to_remove.add(tip_tx.hash)
if to_remove:
self._discard_many(to_remove)
self.log.debug('removed voided txs from tips', txs=[tx.hex() for tx in to_remove])
# Check if any of the txs being confirmed by the voided txs is a tip again. This happens
# if it doesn't have any other valid child.
to_add = set()
for tx_hash in to_remove_parents:
confirmed = False
# check if it has any valid children
meta = not_none(tx_storage.get_metadata(tx_hash))
if meta.voided_by:
continue
children = meta.children
for child_meta in map(tx_storage.get_metadata, children):
assert child_meta is not None
if not child_meta.voided_by:
confirmed = True
break
if not confirmed:
to_add.add(tx_hash)
if to_add:
self._add_many(to_add)
self.log.debug('added txs to tips', txs=[tx.hex() for tx in to_add])
if tx.get_metadata().voided_by:
# this tx is voided, don't need to update the tips
self.log.debug('voided tx, won\'t add it as a tip', tx=tx.hash_hex)
return
self._discard_many(set(tx.parents))
if tx.is_transaction and tx.get_metadata().first_block is None:
assert tx.hash is not None
self._add(tx.hash)
