def validate_new_tx(self, tx: BaseTransaction, skip_block_weight_verification: bool = False) -> bool: """ Process incoming transaction during initialization. These transactions came only from storage. """ assert tx.hash is not None if self.state == self.NodeState.INITIALIZING: if tx.is_genesis: return True else: if tx.is_genesis: raise InvalidNewTransaction('Genesis? {}'.format(tx.hash_hex)) if tx.timestamp - self.reactor.seconds( ) > settings.MAX_FUTURE_TIMESTAMP_ALLOWED: raise InvalidNewTransaction( 'Ignoring transaction in the future {} (timestamp={})'.format( tx.hash_hex, tx.timestamp)) # Verify transaction and raises an TxValidationError if tx is not valid. tx.verify() if tx.is_block: tx = cast(Block, tx) assert tx.hash is not None # XXX: it appears that after casting this assert "casting" is lost if not skip_block_weight_verification: # Validate minimum block difficulty block_weight = self.calculate_block_difficulty(tx) if tx.weight < block_weight - settings.WEIGHT_TOL: raise InvalidNewTransaction( 'Invalid new block {}: weight ({}) is smaller than the minimum weight ({})' .format(tx.hash.hex(), tx.weight, block_weight)) parent_block = tx.get_block_parent() tokens_issued_per_block = self.get_tokens_issued_per_block( parent_block.get_metadata().height + 1) if tx.sum_outputs != tokens_issued_per_block: raise InvalidNewTransaction( 'Invalid number of issued tokens tag=invalid_issued_tokens' ' tx.hash={tx.hash_hex} issued={tx.sum_outputs} allowed={allowed}' .format( tx=tx, allowed=tokens_issued_per_block, )) else: assert tx.hash is not None # XXX: it appears that after casting this assert "casting" is lost # Validate minimum tx difficulty min_tx_weight = self.minimum_tx_weight(tx) if tx.weight < min_tx_weight - settings.WEIGHT_TOL: raise InvalidNewTransaction( 'Invalid new tx {}: weight ({}) is smaller than the minimum weight ({})' .format(tx.hash_hex, tx.weight, min_tx_weight)) return True
def add_tx(self, tx: BaseTransaction) -> None: # if it's a TokenCreationTransaction, update name and symbol self.log.debug('add_tx', tx=tx.hash_hex, ver=tx.version) if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION: from hathor.transaction.token_creation_tx import TokenCreationTransaction tx = cast(TokenCreationTransaction, tx) assert tx.hash is not None self.log.debug('create_token_info', tx=tx.hash_hex, name=tx.token_name, symb=tx.token_symbol) self._create_token_info(tx.hash, tx.token_name, tx.token_symbol) if tx.is_transaction: # Adding this tx to the transactions key list assert isinstance(tx, Transaction) for token_uid in tx.tokens: assert tx.hash is not None self._add_transaction(token_uid, tx.timestamp, tx.hash) for tx_input in tx.inputs: spent_tx = tx.get_spent_tx(tx_input) self._remove_utxo(spent_tx, tx_input.index) for index in range(len(tx.outputs)): self.log.debug('add utxo', tx=tx.hash_hex, index=index) self._add_utxo(tx, index)
def publish_tx(self, tx: BaseTransaction, *, addresses: Optional[Iterable[str]] = None) -> None: """ Publish WALLET_ADDRESS_HISTORY for all addresses of a transaction. """ from hathor.pubsub import HathorEvents if not self.pubsub: return if addresses is None: addresses = tx.get_related_addresses() data = tx.to_json_extended() for address in addresses: self.pubsub.publish(HathorEvents.WALLET_ADDRESS_HISTORY, address=address, history=data)
def minimum_tx_weight(self, tx: BaseTransaction) -> float: """ Returns the minimum weight for the param tx The minimum is calculated by the following function: w = alpha * log(size, 2) + 4.0 + 4.0 ---------------- 1 + k / amount :param tx: tx to calculate the minimum weight :type tx: :py:class:`hathor.transaction.transaction.Transaction` :return: minimum weight for the tx :rtype: float """ # In test mode we don't validate the minimum weight for tx # We do this to allow generating many txs for testing if self.test_mode & TestMode.TEST_TX_WEIGHT: return 1 if tx.is_genesis: return self.min_tx_weight tx_size = len(tx.get_struct()) # We need to take into consideration the decimal places because it is inside the amount. # For instance, if one wants to transfer 20 HTRs, the amount will be 2000. # Max below is preventing division by 0 when handling authority methods that have no outputs amount = max(1, tx.sum_outputs) / (10**settings.DECIMAL_PLACES) weight = (+self.min_tx_weight_coefficient * log(tx_size, 2) + 4 / (1 + self.min_tx_weight_k / amount) + 4) # Make sure the calculated weight is at least the minimum weight = max(weight, self.min_tx_weight) return weight
def add_tx(self, tx: BaseTransaction) -> None: """ Add a new transaction to the index :param tx: Transaction to be added """ assert tx.hash is not None assert tx.storage is not None if tx.hash in self.tx_last_interval: return # Fix the end of the interval of its parents. for parent_hash in tx.parents: pi = self.tx_last_interval.get(parent_hash, None) if not pi: continue if tx.timestamp < pi.end: self.tree.remove(pi) new_interval = Interval(pi.begin, tx.timestamp, pi.data) self.tree.add(new_interval) self.tx_last_interval[parent_hash] = new_interval # Check whether any children has already been added. # It so, the end of the interval is equal to the smallest timestamp of the children. min_timestamp = inf meta = tx.get_metadata() for child_hash in meta.children: if child_hash in self.tx_last_interval: child = tx.storage.get_transaction(child_hash) min_timestamp = min(min_timestamp, child.timestamp) # Add the interval to the tree. interval = Interval(tx.timestamp, min_timestamp, tx.hash) self.tree.add(interval) self.tx_last_interval[tx.hash] = interval
def update_tx(self, tx: BaseTransaction, *, relax_assert: bool = False) -> None: """ Update a tx according to its children. """ assert tx.storage is not None assert tx.hash is not None meta = tx.get_metadata() if meta.voided_by: if not relax_assert: assert tx.hash not in self.tx_last_interval return pi = self.tx_last_interval[tx.hash] min_timestamp = inf for child_hash in meta.children: if child_hash in self.tx_last_interval: child = tx.storage.get_transaction(child_hash) min_timestamp = min(min_timestamp, child.timestamp) if min_timestamp != pi.end: self.tree.remove(pi) new_interval = Interval(pi.begin, min_timestamp, pi.data) self.tree.add(new_interval) self.tx_last_interval[tx.hash] = new_interval
def add_tx(self, tx: BaseTransaction) -> None: """ Checks if this tx has mint or melt inputs/outputs and adds to tokens index """ for tx_input in tx.inputs: spent_tx = tx.get_spent_tx(tx_input) self._remove_from_index(spent_tx, tx_input.index) for index in range(len(tx.outputs)): self._add_to_index(tx, index) # if it's a TokenCreationTransaction, update name and symbol if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION: from hathor.transaction.token_creation_tx import TokenCreationTransaction tx = cast(TokenCreationTransaction, tx) assert tx.hash is not None status = self.tokens[tx.hash] status.name = tx.token_name status.symbol = tx.token_symbol if tx.is_transaction: # Adding this tx to the transactions key list assert isinstance(tx, Transaction) for token_uid in tx.tokens: transactions = self.tokens[token_uid].transactions # It is safe to use the in operator because it is O(log(n)). # http://www.grantjenks.com/docs/sortedcontainers/sortedlist.html#sortedcontainers.SortedList.__contains__ assert tx.hash is not None element = TransactionIndexElement(tx.timestamp, tx.hash) if element in transactions: return transactions.add(element)
def remove_tx(self, tx: BaseTransaction) -> None: """ Remove tx inputs and outputs from the wallet index (indexed by its addresses). """ assert tx.hash is not None addresses = tx.get_related_addresses() for address in addresses: self.index[address].discard(tx.hash)
def remove_tx(self, tx: BaseTransaction) -> None: """ Remove tx inputs and outputs from the wallet index (indexed by its addresses). """ assert tx.hash is not None addresses = tx.get_related_addresses() for address in addresses: self.log.debug('delete address', address=address) self._db.delete((self._cf, self._to_key(address, tx)))
def assert_valid_consensus(self, tx: BaseTransaction) -> None: """Assert the conflict resolution is valid.""" meta = tx.get_metadata() is_tx_executed = bool(not meta.voided_by) for h in meta.conflict_with or []: assert tx.storage is not None conflict_tx = cast(Transaction, tx.storage.get_transaction(h)) conflict_tx_meta = conflict_tx.get_metadata() is_conflict_tx_executed = bool(not conflict_tx_meta.voided_by) assert not (is_tx_executed and is_conflict_tx_executed)
def add_tx(self, tx: BaseTransaction) -> None: """ Add tx inputs and outputs to the wallet index (indexed by its addresses). """ assert tx.hash is not None addresses = tx.get_related_addresses() for address in addresses: self.index[address].add(tx.hash) self.publish_tx(tx, addresses=addresses)
def get_node_label(self, tx: BaseTransaction) -> str: """ Return the node's label for tx. """ assert tx.hash is not None parts = [tx.hash.hex()[-4:]] if self.show_weight: parts.append('w: {:.2f}'.format(tx.weight)) if self.show_acc_weight: meta = tx.get_metadata() parts.append('a: {:.2f}'.format(meta.accumulated_weight)) return '\n'.join(parts)
def add_tx(self, tx: BaseTransaction) -> None: """ Add tx inputs and outputs to the wallet index (indexed by its addresses). """ assert tx.hash is not None addresses = tx.get_related_addresses() for address in addresses: self.log.debug('put address', address=address) self._db.put((self._cf, self._to_key(address, tx)), b'') self.publish_tx(tx, addresses=addresses)
def propagate_tx(self, tx: BaseTransaction, fails_silently: bool = True) -> bool: """Push a new transaction to the network. It is used by both the wallet and the mining modules. :return: True if the transaction was accepted :rtype: bool """ if tx.storage: assert tx.storage == self.tx_storage, 'Invalid tx storage' else: tx.storage = self.tx_storage return self.on_new_tx(tx, fails_silently=fails_silently)
def _remove_from_index(self, tx: BaseTransaction, index: int) -> None: """ Remove tx from mint/melt indexes and total amount """ assert tx.hash is not None tx_output = tx.outputs[index] token_uid = tx.get_token_uid(tx_output.get_token_index()) if tx_output.is_token_authority(): if tx_output.can_mint_token(): # remove from mint index self.tokens[token_uid].mint.discard((tx.hash, index)) if tx_output.can_melt_token(): # remove from melt index self.tokens[token_uid].melt.discard((tx.hash, index)) else: self.tokens[token_uid].total -= tx_output.value
def _add_to_index(self, tx: BaseTransaction, index: int) -> None: """ Add tx to mint/melt indexes and total amount """ assert tx.hash is not None tx_output = tx.outputs[index] token_uid = tx.get_token_uid(tx_output.get_token_index()) if tx_output.is_token_authority(): if tx_output.can_mint_token(): # add to mint index self.tokens[token_uid].mint.add((tx.hash, index)) if tx_output.can_melt_token(): # add to melt index self.tokens[token_uid].melt.add((tx.hash, index)) else: self.tokens[token_uid].total += tx_output.value
def get_node_attrs(self, tx: BaseTransaction) -> Dict[str, str]: """ Return node's attributes. """ assert tx.hash is not None node_attrs = {'label': self.get_node_label(tx)} if tx.is_block: node_attrs.update(self.block_attrs) if tx.is_genesis: node_attrs.update(self.genesis_attrs) meta = tx.get_metadata() if meta.voided_by and len(meta.voided_by) > 0: node_attrs.update(self.voided_attrs) if meta.voided_by and tx.hash in meta.voided_by: node_attrs.update(self.conflict_attrs) return node_attrs
def del_tx(self, tx: BaseTransaction) -> None: for tx_input in tx.inputs: spent_tx = tx.get_spent_tx(tx_input) self._add_utxo(spent_tx, tx_input.index) for index in range(len(tx.outputs)): self._remove_utxo(tx, index) if tx.is_transaction: # Removing this tx from the transactions key list assert isinstance(tx, Transaction) for token_uid in tx.tokens: assert tx.hash is not None self._remove_transaction(token_uid, tx.timestamp, tx.hash) # if it's a TokenCreationTransaction, remove it from index if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION: assert tx.hash is not None self._destroy_token(tx.hash)
def del_tx(self, tx: BaseTransaction) -> None: for tx_input in tx.inputs: spent_tx = tx.get_spent_tx(tx_input) self._add_to_index(spent_tx, tx_input.index) for index in range(len(tx.outputs)): self._remove_from_index(tx, index) if tx.is_transaction: # Removing this tx from the transactions key list assert isinstance(tx, Transaction) for token_uid in tx.tokens: transactions = self._tokens[token_uid]._transactions idx = transactions.bisect_key_left((tx.timestamp, tx.hash)) if idx < len(transactions) and transactions[idx].hash == tx.hash: transactions.pop(idx) # if it's a TokenCreationTransaction, remove it from index if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION: assert tx.hash is not None del self._tokens[tx.hash]
def _remove_utxo(self, tx: BaseTransaction, index: int) -> None: """ Remove tx from mint/melt indexes and total amount """ assert tx.hash is not None tx_output = tx.outputs[index] token_uid = tx.get_token_uid(tx_output.get_token_index()) if tx_output.is_token_authority(): if tx_output.can_mint_token(): # remove from mint index self._remove_authority_utxo(token_uid, tx.hash, index, is_mint=True) if tx_output.can_melt_token(): # remove from melt index self._remove_authority_utxo(token_uid, tx.hash, index, is_mint=False) else: self._subtract_from_total(token_uid, tx_output.value)
def _add_utxo(self, tx: BaseTransaction, index: int) -> None: """ Add tx to mint/melt indexes and total amount """ assert tx.hash is not None tx_output = tx.outputs[index] token_uid = tx.get_token_uid(tx_output.get_token_index()) if tx_output.is_token_authority(): if tx_output.can_mint_token(): # add to mint index self._add_authority_utxo(token_uid, tx.hash, index, is_mint=True) if tx_output.can_melt_token(): # add to melt index self._add_authority_utxo(token_uid, tx.hash, index, is_mint=False) else: self._add_to_total(token_uid, tx_output.value)
def on_new_tx(self, tx: BaseTransaction) -> None: """Checks the inputs and outputs of a transaction for matching keys. If an output matches, will add it to the unspent_txs dict. If an input matches, removes from unspent_txs dict and adds to spent_txs dict. """ assert tx.hash is not None updated = False # check outputs for index, output in enumerate(tx.outputs): script_type_out = parse_address_script(output.script) if script_type_out: if script_type_out.address in self.keys: token_id = tx.get_token_uid(output.get_token_index()) # this wallet received tokens utxo = UnspentTx(tx.hash, index, output.value, tx.timestamp, script_type_out.address, output.token_data, timelock=script_type_out.timelock) self.unspent_txs[token_id][(tx.hash, index)] = utxo # mark key as used self.tokens_received(script_type_out.address) updated = True # publish new output and new balance self.publish_update(HathorEvents.WALLET_OUTPUT_RECEIVED, total=self.get_total_tx(), output=utxo) else: # it's the only one we know, so log warning self.log.warn('unknown script') # check inputs for _input in tx.inputs: assert tx.storage is not None output_tx = tx.storage.get_transaction(_input.tx_id) output = output_tx.outputs[_input.index] token_id = output_tx.get_token_uid(output.get_token_index()) script_type_out = parse_address_script(output.script) if script_type_out: if script_type_out.address in self.keys: # this wallet spent tokens # remove from unspent_txs key = (_input.tx_id, _input.index) old_utxo = self.unspent_txs[token_id].pop(key, None) if old_utxo is None: old_utxo = self.maybe_spent_txs[token_id].pop( key, None) if old_utxo: # add to spent_txs spent = SpentTx(tx.hash, _input.tx_id, _input.index, old_utxo.value, tx.timestamp) self.spent_txs[key].append(spent) updated = True # publish spent output and new balance self.publish_update(HathorEvents.WALLET_INPUT_SPENT, output_spent=spent) else: # If we dont have it in the unspent_txs, it must be in the spent_txs # So we append this spent with the others if key in self.spent_txs: output_tx = tx.storage.get_transaction( _input.tx_id) output = output_tx.outputs[_input.index] spent = SpentTx(tx.hash, _input.tx_id, _input.index, output.value, tx.timestamp) self.spent_txs[key].append(spent) else: self.log.warn('unknown input data') if updated: # TODO update history file? # XXX should wallet always update it or it will be called externally? self.update_balance()
def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = None, quiet: bool = False, fails_silently: bool = True, propagate_to_peers: bool = True, skip_block_weight_verification: bool = False) -> bool: """This method is called when any transaction arrive. If `fails_silently` is False, it may raise either InvalidNewTransaction or TxValidationError. :return: True if the transaction was accepted :rtype: bool """ assert tx.hash is not None if self.state != self.NodeState.INITIALIZING: if self.tx_storage.transaction_exists(tx.hash): if not fails_silently: raise InvalidNewTransaction( 'Transaction already exists {}'.format(tx.hash_hex)) self.log.debug('on_new_tx(): Transaction already exists', tx=tx.hash_hex) return False if self.state != self.NodeState.INITIALIZING or self._full_verification: try: assert self.validate_new_tx( tx, skip_block_weight_verification= skip_block_weight_verification) is True except (InvalidNewTransaction, TxValidationError): # Discard invalid Transaction/block. self.log.debug('tx/block discarded', tx=tx, exc_info=True) if not fails_silently: raise return False if self.state != self.NodeState.INITIALIZING: self.tx_storage.save_transaction(tx) else: self.tx_storage._add_to_cache(tx) if self._full_verification: tx.reset_metadata() else: # When doing a fast init, we don't update the consensus, so we must trust the data on the metadata # For transactions, we don't store them on the tips index if they are voided # We have to execute _add_to_cache before because _del_from_cache does not remove from all indexes metadata = tx.get_metadata() if not tx.is_block and metadata.voided_by: self.tx_storage._del_from_cache(tx) if self.state != self.NodeState.INITIALIZING or self._full_verification: try: tx.update_initial_metadata() self.consensus_algorithm.update(tx) except Exception: self.log.exception('unexpected error when processing tx', tx=tx) self.tx_storage.remove_transaction(tx) raise if not quiet: ts_date = datetime.datetime.fromtimestamp(tx.timestamp) if tx.is_block: self.log.info('new block', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now()) else: self.log.info('new tx', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now()) if propagate_to_peers: # Propagate to our peers. self.connections.send_tx_to_peers(tx) if self.wallet: # TODO Remove it and use pubsub instead. self.wallet.on_new_tx(tx) # Publish to pubsub manager the new tx accepted self.pubsub.publish(HathorEvents.NETWORK_NEW_TX_ACCEPTED, tx=tx) return True
def tx_neighborhood(self, tx: BaseTransaction, format: str = 'pdf', max_level: int = 2, graph_type: str = 'verification') -> Digraph: """ Draw the blocks and transactions around `tx`. :params max_level: Maximum distance between `tx` and the others. :params graph_type: Graph type to be generated. Possibilities are 'verification' and 'funds' """ dot = Digraph(format=format) dot.attr(rankdir='RL') dot.attr('node', shape='oval', style='') root = tx to_visit = [(0, tx)] seen = set([tx.hash]) while to_visit: level, tx = to_visit.pop() assert tx.hash is not None assert tx.storage is not None name = tx.hash.hex() node_attrs = self.get_node_attrs(tx) if tx.hash == root.hash: node_attrs.update(dict(style='filled', penwidth='5.0')) meta = tx.get_metadata() if graph_type == 'verification': if tx.is_block: continue dot.node(name, **node_attrs) if level <= max_level: for h in chain(tx.parents, meta.children): if h not in seen: seen.add(h) tx2 = tx.storage.get_transaction(h) to_visit.append((level + 1, tx2)) for h in tx.parents: if h in seen: dot.edge(name, h.hex()) elif graph_type == 'funds': dot.node(name, **node_attrs) if level <= max_level: spent_outputs_ids = chain.from_iterable( meta.spent_outputs.values()) tx_input_ids = [txin.tx_id for txin in tx.inputs] for h in chain(tx_input_ids, spent_outputs_ids): if h not in seen: seen.add(h) tx2 = tx.storage.get_transaction(h) to_visit.append((level + 1, tx2)) for txin in tx.inputs: if txin.tx_id in seen: dot.edge(name, txin.tx_id.hex()) return dot
def _clone(self, x: BaseTransaction) -> BaseTransaction: if self._clone_if_needed: return x.clone() else: return x
def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = None, quiet: bool = False, fails_silently: bool = True, propagate_to_peers: bool = True) -> bool: """This method is called when any transaction arrive. If `fails_silently` is False, it may raise either InvalidNewTransaction or TxValidationError. :return: True if the transaction was accepted :rtype: bool """ assert tx.hash is not None if self.state != self.NodeState.INITIALIZING: if self.tx_storage.transaction_exists(tx.hash): if not fails_silently: raise InvalidNewTransaction( 'Transaction already exists {}'.format(tx.hash.hex())) self.log.debug( 'on_new_tx(): Already have transaction {}'.format( tx.hash.hex())) return False try: assert self.validate_new_tx(tx) is True except (InvalidNewTransaction, TxValidationError) as e: # Discard invalid Transaction/block. self.log.debug('Transaction/Block discarded', tx=tx, exc=e) if not fails_silently: raise return False if self.state != self.NodeState.INITIALIZING: self.tx_storage.save_transaction(tx) else: tx.reset_metadata() self.tx_storage._add_to_cache(tx) try: tx.update_initial_metadata() self.consensus_algorithm.update(tx) except Exception: pretty_json = json.dumps(tx.to_json(), indent=4) self.log.error( 'An unexpected error occurred when processing {tx.hash_hex}\n' '{pretty_json}', tx=tx, pretty_json=pretty_json) self.tx_storage.remove_transaction(tx) raise if not quiet: ts_date = datetime.datetime.fromtimestamp(tx.timestamp) if tx.is_block: self.log.info('New block found', tag='new_block', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now()) else: self.log.info('New transaction found', tag='new_tx', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now()) if propagate_to_peers: # Propagate to our peers. self.connections.send_tx_to_peers(tx) if self.wallet: # TODO Remove it and use pubsub instead. self.wallet.on_new_tx(tx) # Publish to pubsub manager the new tx accepted self.pubsub.publish(HathorEvents.NETWORK_NEW_TX_ACCEPTED, tx=tx) return True
def _score_block_dfs(self, block: BaseTransaction, used: Set[bytes], mark_as_best_chain: bool, newest_timestamp: int) -> float: """ Internal method to run a DFS. It is used by `calculate_score()`. """ assert block.storage is not None assert block.hash is not None assert block.is_block storage = block.storage from hathor.transaction import Block score = block.weight for parent in block.get_parents(): if parent.is_block: assert isinstance(parent, Block) if parent.timestamp <= newest_timestamp: meta = parent.get_metadata() x = meta.score else: x = self._score_block_dfs(parent, used, mark_as_best_chain, newest_timestamp) score = sum_weights(score, x) else: from hathor.transaction.storage.traversal import BFSWalk bfs = BFSWalk(storage, is_dag_verifications=True, is_left_to_right=False) for tx in bfs.run(parent, skip_root=False): assert tx.hash is not None assert not tx.is_block if tx.hash in used: bfs.skip_neighbors(tx) continue used.add(tx.hash) meta = tx.get_metadata() if meta.first_block: first_block = storage.get_transaction(meta.first_block) if first_block.timestamp <= newest_timestamp: bfs.skip_neighbors(tx) continue if mark_as_best_chain: assert meta.first_block is None meta.first_block = block.hash storage.save_transaction(tx, only_metadata=True) score = sum_weights(score, tx.weight) # Always save the score when it is calculated. meta = block.get_metadata() if not meta.score: meta.score = score storage.save_transaction(block, only_metadata=True) else: # The score of a block is immutable since the sub-DAG behind it is immutable as well. # Thus, if we have already calculated it, we just check the consistency of the calculation. # Unfortunately we may have to calculate it more than once when a new block arrives in a side # side because the `first_block` points only to the best chain. assert abs(meta.score - score) < 1e-10, \ 'hash={} meta.score={} score={}'.format(block.hash.hex(), meta.score, score) return score
def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = None, quiet: bool = False, fails_silently: bool = True, propagate_to_peers: bool = True, skip_block_weight_verification: bool = False, sync_checkpoints: bool = False, partial: bool = False) -> bool: """ New method for adding transactions or blocks that steps the validation state machine. :param tx: transaction to be added :param conn: optionally specify the protocol instance where this tx was received from :param quiet: if True will not log when a new tx is accepted :param fails_silently: if False will raise an exception when tx cannot be added :param propagate_to_peers: if True will relay the tx to other peers if it is accepted :param skip_block_weight_verification: if True will not check the tx PoW :param sync_checkpoints: if True and also partial=True, will try to validate as a checkpoint and set the proper validation state, this is used for adding txs from the sync-checkpoints phase :param partial: if True will accept txs that can't be fully validated yet (because of missing parent/input) but will run a basic validation of what can be validated (PoW and other basic fields) """ assert tx.hash is not None if self.tx_storage.transaction_exists(tx.hash): if not fails_silently: raise InvalidNewTransaction( 'Transaction already exists {}'.format(tx.hash_hex)) self.log.warn('on_new_tx(): Transaction already exists', tx=tx.hash_hex) return False if tx.timestamp - self.reactor.seconds( ) > settings.MAX_FUTURE_TIMESTAMP_ALLOWED: if not fails_silently: raise InvalidNewTransaction( 'Ignoring transaction in the future {} (timestamp={})'. format(tx.hash_hex, tx.timestamp)) self.log.warn('on_new_tx(): Ignoring transaction in the future', tx=tx.hash_hex, future_timestamp=tx.timestamp) return False tx.storage = self.tx_storage try: metadata = tx.get_metadata() except TransactionDoesNotExist: if not fails_silently: raise InvalidNewTransaction('missing parent') self.log.warn('on_new_tx(): missing parent', tx=tx.hash_hex) return False if metadata.validation.is_invalid(): if not fails_silently: raise InvalidNewTransaction('previously marked as invalid') self.log.warn('on_new_tx(): previously marked as invalid', tx=tx.hash_hex) return False # if partial=False (the default) we don't even try to partially validate transactions if not partial or (metadata.validation.is_fully_connected() or tx.can_validate_full()): if isinstance(tx, Transaction) and self.tx_storage.is_tx_needed( tx.hash): tx._height_cache = self.tx_storage.needed_index_height(tx.hash) if not metadata.validation.is_fully_connected(): try: tx.validate_full(sync_checkpoints=sync_checkpoints) except HathorError as e: if not fails_silently: raise InvalidNewTransaction( 'full validation failed') from e self.log.warn('on_new_tx(): full validation failed', tx=tx.hash_hex, exc_info=True) return False # The method below adds the tx as a child of the parents # This needs to be called right before the save because we were adding the children # in the tx parents even if the tx was invalid (failing the verifications above) # then I would have a children that was not in the storage tx.update_initial_metadata() self.tx_storage.save_transaction(tx, add_to_indexes=True) try: self.consensus_algorithm.update(tx) except HathorError as e: if not fails_silently: raise InvalidNewTransaction( 'consensus update failed') from e self.log.warn('on_new_tx(): consensus update failed', tx=tx.hash_hex) return False else: assert tx.validate_full(skip_block_weight_verification=True) self.tx_fully_validated(tx) elif sync_checkpoints: metadata.children = self.tx_storage.children_from_deps(tx.hash) try: tx.validate_checkpoint(self.checkpoints) except HathorError: if not fails_silently: raise InvalidNewTransaction('checkpoint validation failed') self.log.warn('on_new_tx(): checkpoint validation failed', tx=tx.hash_hex, exc_info=True) return False self.tx_storage.save_transaction(tx) self.tx_storage.add_to_deps_index(tx.hash, tx.get_all_dependencies()) self.tx_storage.add_needed_deps(tx) else: if isinstance(tx, Block) and not tx.has_basic_block_parent(): if not fails_silently: raise InvalidNewTransaction( 'block parent needs to be at least basic-valid') self.log.warn( 'on_new_tx(): block parent needs to be at least basic-valid', tx=tx.hash_hex) return False if not tx.validate_basic(): if not fails_silently: raise InvalidNewTransaction('basic validation failed') self.log.warn('on_new_tx(): basic validation failed', tx=tx.hash_hex) return False # The method below adds the tx as a child of the parents # This needs to be called right before the save because we were adding the children # in the tx parents even if the tx was invalid (failing the verifications above) # then I would have a children that was not in the storage tx.update_initial_metadata() self.tx_storage.save_transaction(tx) self.tx_storage.add_to_deps_index(tx.hash, tx.get_all_dependencies()) self.tx_storage.add_needed_deps(tx) if tx.is_transaction: self.tx_storage.remove_from_needed_index(tx.hash) try: self.step_validations([tx]) except (AssertionError, HathorError) as e: if not fails_silently: raise InvalidNewTransaction('step validations failed') from e self.log.warn('on_new_tx(): step validations failed', tx=tx.hash_hex, exc_info=True) return False if not quiet: ts_date = datetime.datetime.fromtimestamp(tx.timestamp) now = datetime.datetime.fromtimestamp(self.reactor.seconds()) if tx.is_block: self.log.info('new block', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now(now)) else: self.log.info('new tx', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now(now)) if propagate_to_peers: # Propagate to our peers. self.connections.send_tx_to_peers(tx) return True
def send_data(self, tx: BaseTransaction) -> None: """ Send a DATA message. """ # self.log.debug('Sending {tx.hash_hex}...', tx=tx) payload = base64.b64encode(tx.get_struct()).decode('ascii') self.send_message(ProtocolMessages.DATA, payload)
def update(self, tx: BaseTransaction) -> None: assert tx.hash is not None assert tx.storage is not None to_remove: Set[bytes] = set() to_remove_parents: Set[bytes] = set() tx_storage = tx.storage for tip_tx in self.iter(tx_storage): assert tip_tx.hash is not None meta = tip_tx.get_metadata() # a new tx/block added might cause a tx in the tips to become voided. For instance, there might be a tx1 a # double spending tx2, where tx1 is valid and tx2 voided. A new block confirming tx2 will make it valid # while tx1 becomes voided, so it has to be removed from the tips. The txs confirmed by tx1 need to be # double checked, as they might themselves become tips (hence we use to_remove_parents) if meta.voided_by: to_remove.add(tip_tx.hash) to_remove_parents.update(tip_tx.parents) continue # might also happen that a tip has a child that became valid, so it's not a tip anymore confirmed = False for child_meta in map(tx_storage.get_metadata, meta.children): assert child_meta is not None if not child_meta.voided_by: confirmed = True break if confirmed: to_remove.add(tip_tx.hash) if to_remove: self._discard_many(to_remove) self.log.debug('removed voided txs from tips', txs=[tx.hex() for tx in to_remove]) # Check if any of the txs being confirmed by the voided txs is a tip again. This happens # if it doesn't have any other valid child. to_add = set() for tx_hash in to_remove_parents: confirmed = False # check if it has any valid children meta = not_none(tx_storage.get_metadata(tx_hash)) if meta.voided_by: continue children = meta.children for child_meta in map(tx_storage.get_metadata, children): assert child_meta is not None if not child_meta.voided_by: confirmed = True break if not confirmed: to_add.add(tx_hash) if to_add: self._add_many(to_add) self.log.debug('added txs to tips', txs=[tx.hex() for tx in to_add]) if tx.get_metadata().voided_by: # this tx is voided, don't need to update the tips self.log.debug('voided tx, won\'t add it as a tip', tx=tx.hash_hex) return self._discard_many(set(tx.parents)) if tx.is_transaction and tx.get_metadata().first_block is None: assert tx.hash is not None self._add(tx.hash)