def update_tx(self, tx: BaseTransaction, *, relax_assert: bool = False) -> None: """ Update a tx according to its children. """ assert tx.storage is not None assert tx.hash is not None meta = tx.get_metadata() if meta.voided_by: if not relax_assert: assert tx.hash not in self.tx_last_interval return pi = self.tx_last_interval[tx.hash] min_timestamp = inf for child_hash in meta.children: if child_hash in self.tx_last_interval: child = tx.storage.get_transaction(child_hash) min_timestamp = min(min_timestamp, child.timestamp) if min_timestamp != pi.end: self.tree.remove(pi) new_interval = Interval(pi.begin, min_timestamp, pi.data) self.tree.add(new_interval) self.tx_last_interval[tx.hash] = new_interval
def add_tx(self, tx: BaseTransaction) -> None: """ Add a new transaction to the index :param tx: Transaction to be added """ assert tx.hash is not None assert tx.storage is not None if tx.hash in self.tx_last_interval: return # Fix the end of the interval of its parents. for parent_hash in tx.parents: pi = self.tx_last_interval.get(parent_hash, None) if not pi: continue if tx.timestamp < pi.end: self.tree.remove(pi) new_interval = Interval(pi.begin, tx.timestamp, pi.data) self.tree.add(new_interval) self.tx_last_interval[parent_hash] = new_interval # Check whether any children has already been added. # It so, the end of the interval is equal to the smallest timestamp of the children. min_timestamp = inf meta = tx.get_metadata() for child_hash in meta.children: if child_hash in self.tx_last_interval: child = tx.storage.get_transaction(child_hash) min_timestamp = min(min_timestamp, child.timestamp) # Add the interval to the tree. interval = Interval(tx.timestamp, min_timestamp, tx.hash) self.tree.add(interval) self.tx_last_interval[tx.hash] = interval
def assert_valid_consensus(self, tx: BaseTransaction) -> None: """Assert the conflict resolution is valid.""" meta = tx.get_metadata() is_tx_executed = bool(not meta.voided_by) for h in meta.conflict_with or []: assert tx.storage is not None conflict_tx = cast(Transaction, tx.storage.get_transaction(h)) conflict_tx_meta = conflict_tx.get_metadata() is_conflict_tx_executed = bool(not conflict_tx_meta.voided_by) assert not (is_tx_executed and is_conflict_tx_executed)
def get_node_label(self, tx: BaseTransaction) -> str: """ Return the node's label for tx. """ assert tx.hash is not None parts = [tx.hash.hex()[-4:]] if self.show_weight: parts.append('w: {:.2f}'.format(tx.weight)) if self.show_acc_weight: meta = tx.get_metadata() parts.append('a: {:.2f}'.format(meta.accumulated_weight)) return '\n'.join(parts)
def get_node_attrs(self, tx: BaseTransaction) -> Dict[str, str]: """ Return node's attributes. """ assert tx.hash is not None node_attrs = {'label': self.get_node_label(tx)} if tx.is_block: node_attrs.update(self.block_attrs) if tx.is_genesis: node_attrs.update(self.genesis_attrs) meta = tx.get_metadata() if meta.voided_by and len(meta.voided_by) > 0: node_attrs.update(self.voided_attrs) if meta.voided_by and tx.hash in meta.voided_by: node_attrs.update(self.conflict_attrs) return node_attrs
def tx_neighborhood(self, tx: BaseTransaction, format: str = 'pdf', max_level: int = 2, graph_type: str = 'verification') -> Digraph: """ Draw the blocks and transactions around `tx`. :params max_level: Maximum distance between `tx` and the others. :params graph_type: Graph type to be generated. Possibilities are 'verification' and 'funds' """ dot = Digraph(format=format) dot.attr(rankdir='RL') dot.attr('node', shape='oval', style='') root = tx to_visit = [(0, tx)] seen = set([tx.hash]) while to_visit: level, tx = to_visit.pop() assert tx.hash is not None assert tx.storage is not None name = tx.hash.hex() node_attrs = self.get_node_attrs(tx) if tx.hash == root.hash: node_attrs.update(dict(style='filled', penwidth='5.0')) meta = tx.get_metadata() if graph_type == 'verification': if tx.is_block: continue dot.node(name, **node_attrs) if level <= max_level: for h in chain(tx.parents, meta.children): if h not in seen: seen.add(h) tx2 = tx.storage.get_transaction(h) to_visit.append((level + 1, tx2)) for h in tx.parents: if h in seen: dot.edge(name, h.hex()) elif graph_type == 'funds': dot.node(name, **node_attrs) if level <= max_level: spent_outputs_ids = chain.from_iterable( meta.spent_outputs.values()) tx_input_ids = [txin.tx_id for txin in tx.inputs] for h in chain(tx_input_ids, spent_outputs_ids): if h not in seen: seen.add(h) tx2 = tx.storage.get_transaction(h) to_visit.append((level + 1, tx2)) for txin in tx.inputs: if txin.tx_id in seen: dot.edge(name, txin.tx_id.hex()) return dot
def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = None, quiet: bool = False, fails_silently: bool = True, propagate_to_peers: bool = True, skip_block_weight_verification: bool = False, sync_checkpoints: bool = False, partial: bool = False) -> bool: """ New method for adding transactions or blocks that steps the validation state machine. :param tx: transaction to be added :param conn: optionally specify the protocol instance where this tx was received from :param quiet: if True will not log when a new tx is accepted :param fails_silently: if False will raise an exception when tx cannot be added :param propagate_to_peers: if True will relay the tx to other peers if it is accepted :param skip_block_weight_verification: if True will not check the tx PoW :param sync_checkpoints: if True and also partial=True, will try to validate as a checkpoint and set the proper validation state, this is used for adding txs from the sync-checkpoints phase :param partial: if True will accept txs that can't be fully validated yet (because of missing parent/input) but will run a basic validation of what can be validated (PoW and other basic fields) """ assert tx.hash is not None if self.tx_storage.transaction_exists(tx.hash): if not fails_silently: raise InvalidNewTransaction( 'Transaction already exists {}'.format(tx.hash_hex)) self.log.warn('on_new_tx(): Transaction already exists', tx=tx.hash_hex) return False if tx.timestamp - self.reactor.seconds( ) > settings.MAX_FUTURE_TIMESTAMP_ALLOWED: if not fails_silently: raise InvalidNewTransaction( 'Ignoring transaction in the future {} (timestamp={})'. format(tx.hash_hex, tx.timestamp)) self.log.warn('on_new_tx(): Ignoring transaction in the future', tx=tx.hash_hex, future_timestamp=tx.timestamp) return False tx.storage = self.tx_storage try: metadata = tx.get_metadata() except TransactionDoesNotExist: if not fails_silently: raise InvalidNewTransaction('missing parent') self.log.warn('on_new_tx(): missing parent', tx=tx.hash_hex) return False if metadata.validation.is_invalid(): if not fails_silently: raise InvalidNewTransaction('previously marked as invalid') self.log.warn('on_new_tx(): previously marked as invalid', tx=tx.hash_hex) return False # if partial=False (the default) we don't even try to partially validate transactions if not partial or (metadata.validation.is_fully_connected() or tx.can_validate_full()): if isinstance(tx, Transaction) and self.tx_storage.is_tx_needed( tx.hash): tx._height_cache = self.tx_storage.needed_index_height(tx.hash) if not metadata.validation.is_fully_connected(): try: tx.validate_full(sync_checkpoints=sync_checkpoints) except HathorError as e: if not fails_silently: raise InvalidNewTransaction( 'full validation failed') from e self.log.warn('on_new_tx(): full validation failed', tx=tx.hash_hex, exc_info=True) return False # The method below adds the tx as a child of the parents # This needs to be called right before the save because we were adding the children # in the tx parents even if the tx was invalid (failing the verifications above) # then I would have a children that was not in the storage tx.update_initial_metadata() self.tx_storage.save_transaction(tx, add_to_indexes=True) try: self.consensus_algorithm.update(tx) except HathorError as e: if not fails_silently: raise InvalidNewTransaction( 'consensus update failed') from e self.log.warn('on_new_tx(): consensus update failed', tx=tx.hash_hex) return False else: assert tx.validate_full(skip_block_weight_verification=True) self.tx_fully_validated(tx) elif sync_checkpoints: metadata.children = self.tx_storage.children_from_deps(tx.hash) try: tx.validate_checkpoint(self.checkpoints) except HathorError: if not fails_silently: raise InvalidNewTransaction('checkpoint validation failed') self.log.warn('on_new_tx(): checkpoint validation failed', tx=tx.hash_hex, exc_info=True) return False self.tx_storage.save_transaction(tx) self.tx_storage.add_to_deps_index(tx.hash, tx.get_all_dependencies()) self.tx_storage.add_needed_deps(tx) else: if isinstance(tx, Block) and not tx.has_basic_block_parent(): if not fails_silently: raise InvalidNewTransaction( 'block parent needs to be at least basic-valid') self.log.warn( 'on_new_tx(): block parent needs to be at least basic-valid', tx=tx.hash_hex) return False if not tx.validate_basic(): if not fails_silently: raise InvalidNewTransaction('basic validation failed') self.log.warn('on_new_tx(): basic validation failed', tx=tx.hash_hex) return False # The method below adds the tx as a child of the parents # This needs to be called right before the save because we were adding the children # in the tx parents even if the tx was invalid (failing the verifications above) # then I would have a children that was not in the storage tx.update_initial_metadata() self.tx_storage.save_transaction(tx) self.tx_storage.add_to_deps_index(tx.hash, tx.get_all_dependencies()) self.tx_storage.add_needed_deps(tx) if tx.is_transaction: self.tx_storage.remove_from_needed_index(tx.hash) try: self.step_validations([tx]) except (AssertionError, HathorError) as e: if not fails_silently: raise InvalidNewTransaction('step validations failed') from e self.log.warn('on_new_tx(): step validations failed', tx=tx.hash_hex, exc_info=True) return False if not quiet: ts_date = datetime.datetime.fromtimestamp(tx.timestamp) now = datetime.datetime.fromtimestamp(self.reactor.seconds()) if tx.is_block: self.log.info('new block', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now(now)) else: self.log.info('new tx', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now(now)) if propagate_to_peers: # Propagate to our peers. self.connections.send_tx_to_peers(tx) return True
def _score_block_dfs(self, block: BaseTransaction, used: Set[bytes], mark_as_best_chain: bool, newest_timestamp: int) -> float: """ Internal method to run a DFS. It is used by `calculate_score()`. """ assert block.storage is not None assert block.hash is not None assert block.is_block storage = block.storage from hathor.transaction import Block score = block.weight for parent in block.get_parents(): if parent.is_block: assert isinstance(parent, Block) if parent.timestamp <= newest_timestamp: meta = parent.get_metadata() x = meta.score else: x = self._score_block_dfs(parent, used, mark_as_best_chain, newest_timestamp) score = sum_weights(score, x) else: from hathor.transaction.storage.traversal import BFSWalk bfs = BFSWalk(storage, is_dag_verifications=True, is_left_to_right=False) for tx in bfs.run(parent, skip_root=False): assert tx.hash is not None assert not tx.is_block if tx.hash in used: bfs.skip_neighbors(tx) continue used.add(tx.hash) meta = tx.get_metadata() if meta.first_block: first_block = storage.get_transaction(meta.first_block) if first_block.timestamp <= newest_timestamp: bfs.skip_neighbors(tx) continue if mark_as_best_chain: assert meta.first_block is None meta.first_block = block.hash storage.save_transaction(tx, only_metadata=True) score = sum_weights(score, tx.weight) # Always save the score when it is calculated. meta = block.get_metadata() if not meta.score: meta.score = score storage.save_transaction(block, only_metadata=True) else: # The score of a block is immutable since the sub-DAG behind it is immutable as well. # Thus, if we have already calculated it, we just check the consistency of the calculation. # Unfortunately we may have to calculate it more than once when a new block arrives in a side # side because the `first_block` points only to the best chain. assert abs(meta.score - score) < 1e-10, \ 'hash={} meta.score={} score={}'.format(block.hash.hex(), meta.score, score) return score
def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = None, quiet: bool = False, fails_silently: bool = True, propagate_to_peers: bool = True, skip_block_weight_verification: bool = False) -> bool: """This method is called when any transaction arrive. If `fails_silently` is False, it may raise either InvalidNewTransaction or TxValidationError. :return: True if the transaction was accepted :rtype: bool """ assert tx.hash is not None if self.state != self.NodeState.INITIALIZING: if self.tx_storage.transaction_exists(tx.hash): if not fails_silently: raise InvalidNewTransaction( 'Transaction already exists {}'.format(tx.hash_hex)) self.log.debug('on_new_tx(): Transaction already exists', tx=tx.hash_hex) return False if self.state != self.NodeState.INITIALIZING or self._full_verification: try: assert self.validate_new_tx( tx, skip_block_weight_verification= skip_block_weight_verification) is True except (InvalidNewTransaction, TxValidationError): # Discard invalid Transaction/block. self.log.debug('tx/block discarded', tx=tx, exc_info=True) if not fails_silently: raise return False if self.state != self.NodeState.INITIALIZING: self.tx_storage.save_transaction(tx) else: self.tx_storage._add_to_cache(tx) if self._full_verification: tx.reset_metadata() else: # When doing a fast init, we don't update the consensus, so we must trust the data on the metadata # For transactions, we don't store them on the tips index if they are voided # We have to execute _add_to_cache before because _del_from_cache does not remove from all indexes metadata = tx.get_metadata() if not tx.is_block and metadata.voided_by: self.tx_storage._del_from_cache(tx) if self.state != self.NodeState.INITIALIZING or self._full_verification: try: tx.update_initial_metadata() self.consensus_algorithm.update(tx) except Exception: self.log.exception('unexpected error when processing tx', tx=tx) self.tx_storage.remove_transaction(tx) raise if not quiet: ts_date = datetime.datetime.fromtimestamp(tx.timestamp) if tx.is_block: self.log.info('new block', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now()) else: self.log.info('new tx', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now()) if propagate_to_peers: # Propagate to our peers. self.connections.send_tx_to_peers(tx) if self.wallet: # TODO Remove it and use pubsub instead. self.wallet.on_new_tx(tx) # Publish to pubsub manager the new tx accepted self.pubsub.publish(HathorEvents.NETWORK_NEW_TX_ACCEPTED, tx=tx) return True
def update(self, tx: BaseTransaction) -> None: assert tx.hash is not None assert tx.storage is not None to_remove: Set[bytes] = set() to_remove_parents: Set[bytes] = set() tx_storage = tx.storage for tip_tx in self.iter(tx_storage): assert tip_tx.hash is not None meta = tip_tx.get_metadata() # a new tx/block added might cause a tx in the tips to become voided. For instance, there might be a tx1 a # double spending tx2, where tx1 is valid and tx2 voided. A new block confirming tx2 will make it valid # while tx1 becomes voided, so it has to be removed from the tips. The txs confirmed by tx1 need to be # double checked, as they might themselves become tips (hence we use to_remove_parents) if meta.voided_by: to_remove.add(tip_tx.hash) to_remove_parents.update(tip_tx.parents) continue # might also happen that a tip has a child that became valid, so it's not a tip anymore confirmed = False for child_meta in map(tx_storage.get_metadata, meta.children): assert child_meta is not None if not child_meta.voided_by: confirmed = True break if confirmed: to_remove.add(tip_tx.hash) if to_remove: self._discard_many(to_remove) self.log.debug('removed voided txs from tips', txs=[tx.hex() for tx in to_remove]) # Check if any of the txs being confirmed by the voided txs is a tip again. This happens # if it doesn't have any other valid child. to_add = set() for tx_hash in to_remove_parents: confirmed = False # check if it has any valid children meta = not_none(tx_storage.get_metadata(tx_hash)) if meta.voided_by: continue children = meta.children for child_meta in map(tx_storage.get_metadata, children): assert child_meta is not None if not child_meta.voided_by: confirmed = True break if not confirmed: to_add.add(tx_hash) if to_add: self._add_many(to_add) self.log.debug('added txs to tips', txs=[tx.hex() for tx in to_add]) if tx.get_metadata().voided_by: # this tx is voided, don't need to update the tips self.log.debug('voided tx, won\'t add it as a tip', tx=tx.hash_hex) return self._discard_many(set(tx.parents)) if tx.is_transaction and tx.get_metadata().first_block is None: assert tx.hash is not None self._add(tx.hash)