def create_token_commitment_for_eon(token: Token, eon_number): logger.info('Creating Token Commitment for {} at {}'.format( token.address, eon_number)) last_eon_number = eon_number - 1 with transaction.atomic(): wallets = Wallet.objects\ .filter( token=token, registration_operator_authorization__isnull=False, trail_identifier__isnull=False)\ .order_by('trail_identifier') new_balances = [] left, right = 0, 0 for wallet in wallets: with wallet.lock(auto_renewal=True): wallet_transfer_context = WalletTransferContext( wallet=wallet, transfer=None) last_transfer, last_transfer_is_outgoing = wallet_transfer_context.last_appended_active_transfer( eon_number=last_eon_number) last_transfer_active_state = None if last_transfer is not None and last_transfer.is_open_swap(): last_transfer.retire_swap() if last_transfer is not None: last_transfer_active_state = WalletTransferContext.appropriate_transfer_active_state( transfer=last_transfer, is_outgoing=last_transfer_is_outgoing) available_funds = wallet_transfer_context.available_funds_at_eon( eon_number=last_eon_number, only_appended=True) right = left + available_funds assert right >= left, 'Wallet {} Token {} Balance {}'.format( wallet.address, token.address, available_funds) passive_checksum, passive_amount, passive_marker = wallet_transfer_context.get_passive_values( eon_number=last_eon_number) new_balances.append({ 'contract': settings.HUB_LQD_CONTRACT_ADDRESS, 'token': token.address, 'wallet': wallet.address, 'left': left, 'right': right, 'active_state_checksum': last_transfer_active_state.checksum() if last_transfer_active_state is not None else b'\0'*32, 'active_state': last_transfer_active_state, 'passive_checksum': passive_checksum, 'passive_amount': passive_amount, 'passive_marker': passive_marker, }) left = right last_incoming_passive_transfer = wallet_transfer_context.last_appended_incoming_passive_transfer( eon_number=last_eon_number) if last_incoming_passive_transfer: wallet_transfer_context = WalletTransferContext( wallet=wallet, transfer=last_incoming_passive_transfer) passive_eon_transfers_list = wallet_transfer_context.incoming_passive_transfers_list( only_appended=True, force_append=False) passive_transfers_merkle_tree = wallet_transfer_context.incoming_passive_transfers_tree( only_appended=True, force_append=False) for index, incoming_passive_transfer in enumerate(passive_eon_transfers_list): final_transfer_index = index final_transfer_membership_proof = passive_transfers_merkle_tree.proof( final_transfer_index) final_transfer_membership_proof_chain = final_transfer_membership_proof.get( "chain") final_transfer_membership_proof_values = final_transfer_membership_proof.get( "values") assert incoming_passive_transfer.final_receipt_hashes is None assert incoming_passive_transfer.final_receipt_index is None assert incoming_passive_transfer.final_receipt_values is None incoming_passive_transfer.final_receipt_hashes = final_transfer_membership_proof_chain incoming_passive_transfer.final_receipt_index = final_transfer_index incoming_passive_transfer.final_receipt_values = final_transfer_membership_proof_values incoming_passive_transfer.save() if last_transfer_active_state is None: continue wallet_transfer_context = WalletTransferContext( wallet=wallet, transfer=last_transfer) starting_balance = int( wallet_transfer_context.starting_balance_in_eon(last_eon_number)) # if last active transfer is a multi eon swap # starting balance included in every tx checksum should be set to the cached starting balance # this way checkpoint state will match signed active state if last_transfer.is_swap() and not last_transfer.cancelled: if Transfer.objects.filter(eon_number=last_transfer.eon_number-1, tx_id=last_transfer.tx_id).exists(): matched_out, matched_in = last_transfer.matched_amounts( all_eons=True) current_matched_out, current_matched_in = last_transfer.matched_amounts( all_eons=False) if last_transfer_is_outgoing: sender_starting_balance = last_transfer.sender_starting_balance # current eon's starting balance should be equal to # cached starting balance - committed matched out amount in past rounds assert(starting_balance == sender_starting_balance - matched_out + current_matched_out) starting_balance = sender_starting_balance else: recipient_starting_balance = last_transfer.recipient_starting_balance # current eon's starting balance should be equal to # cached starting balance + committed matched in amount in past rounds assert( starting_balance == recipient_starting_balance + matched_in - current_matched_in) starting_balance = recipient_starting_balance confirmed_eon_transfers_list = wallet_transfer_context.authorized_transfers_list( only_appended=True, force_append=False) confirmed_eon_transfers_list_shorthand = wallet_transfer_context.authorized_transfers_list_shorthand( only_appended=True, force_append=False, last_transfer_is_finalized=False, starting_balance=starting_balance) transaction_merkle_tree = TransactionMerkleTree( confirmed_eon_transfers_list_shorthand) transaction_merkle_tree_root = hex_value( transaction_merkle_tree.root_hash()) assert transaction_merkle_tree_root == last_transfer_active_state.tx_set_hash,\ '{}/{}'.format(transaction_merkle_tree_root, last_transfer_active_state.tx_set_hash) for confirmed_incoming_transfer in confirmed_eon_transfers_list: if confirmed_incoming_transfer.recipient != wallet: continue final_transfer_index = transaction_merkle_tree.merkle_tree_nonce_map.get( confirmed_incoming_transfer.nonce) final_transfer_membership_proof_chain = transaction_merkle_tree.proof( final_transfer_index) assert confirmed_incoming_transfer.final_receipt_hashes is None assert confirmed_incoming_transfer.final_receipt_index is None confirmed_incoming_transfer.final_receipt_hashes = final_transfer_membership_proof_chain confirmed_incoming_transfer.final_receipt_index = final_transfer_index confirmed_incoming_transfer.save() managed_funds = 0 if eon_number > 1: last_eon = LocalViewInterface.confirmed(eon_number=last_eon_number) pending_withdrawals_until_last_eon = \ WithdrawalRequest.objects\ .filter(wallet__token=token, eon_number__lte=last_eon_number, slashed=False)\ .filter(Q(withdrawal__isnull=True) | Q(withdrawal__block__gt=last_eon.block)) if not pending_withdrawals_until_last_eon.exists(): last_eon_pending_withdrawals = 0 else: last_eon_pending_withdrawals = pending_withdrawals_until_last_eon\ .aggregate(Sum('amount')) \ .get('amount__sum') total_token_balance = last_eon.contractledgerstate_set.get( token=token).total_balance managed_funds = total_token_balance - last_eon_pending_withdrawals if right < managed_funds: logger.warning('UNCLAIMED FUNDS: {} in {}'.format( managed_funds - right, token.address)) send_admin_email( subject='Soft TokenCommitment Warning: Extra funds', content='There are some additional funds in the balance pool that belong to no one: {} of {}' .format(managed_funds - right, token.address)) altered_balances = new_balances + [{ 'contract': settings.HUB_LQD_CONTRACT_ADDRESS, 'token': token.address, 'wallet': settings.HUB_OWNER_ACCOUNT_ADDRESS, 'left': left, 'right': managed_funds, 'active_state_checksum': b'\0'*32, 'active_state': None, 'passive_checksum': b'\0'*32, 'passive_amount': 0, 'passive_marker': 0, }] new_merkle_tree = MerkleTree(altered_balances, managed_funds) right = managed_funds else: if right > managed_funds: logger.error('OVERCLAIMING FUNDS!! {} > {} in {}'.format( right, managed_funds, token.address)) send_admin_email( subject='HARD Checkpoint Error: OVERCLAIMING!', content='OVERCLAIMING FUNDS!! {} > {} in {}'.format(right, managed_funds, token.address)) new_merkle_tree = MerkleTree(new_balances, right) bulk_manager = BulkCreateManager(chunk_size=500) for index, balance in enumerate(new_balances): if not balance.get('wallet') or balance.get('wallet') == '0x0000000000000000000000000000000000000000': continue merkle_proof = new_merkle_tree.proof(index) wallet = Wallet.objects.get( token=token, address=remove_0x_prefix(balance.get('wallet'))) # TODO verify validity through RPC prior to insertion assert(wallet.trail_identifier == index) # create records in batches bulk_manager.add( ExclusiveBalanceAllotment( wallet=wallet, eon_number=eon_number, left=balance.get('left'), right=balance.get('right'), merkle_proof_hashes=merkle_proof.get('chain'), merkle_proof_values=merkle_proof.get('values'), merkle_proof_trail=index, active_state=balance.get('active_state') ) ) # make sure remaining batch is added bulk_manager.done() token_commitment = TokenCommitment.objects.create( token=token, merkle_root=hex_value(new_merkle_tree.root_hash()), upper_bound=right) return token_commitment
def create(self, validated_data): active_state_signature_data = validated_data.pop('debit_signature') wallet = validated_data.pop('wallet') recipient = validated_data.pop('recipient') # get current eon current_eon = LocalViewInterface.latest().eon_number() # transfer eon should be the current eon number if validated_data.pop('eon_number') != current_eon: raise serializers.ValidationError( detail='', code=ErrorCode.EON_NUMBER_OUT_OF_SYNC) # TODO refactor this such that the recipient is only locked after the sender's details are verified wallets = sorted([wallet, recipient], key=lambda w: w.trail_identifier) with RootCommitment.read_write_lock( suffix=current_eon, auto_renewal=False), wallets[0].lock( auto_renewal=False), wallets[1].lock(auto_renewal=False): if RootCommitment.objects.filter(eon_number=current_eon + 1).exists(): raise serializers.ValidationError( detail='', code=ErrorCode.EON_NUMBER_OUT_OF_SYNC) transfer = Transfer(wallet=wallet, amount=validated_data.pop('amount'), eon_number=current_eon, recipient=recipient, nonce=validated_data.pop('nonce'), passive=True) wallet_view_context = WalletTransferContext(wallet=wallet, transfer=transfer) recipient_view_context = WalletTransferContext(wallet=recipient, transfer=transfer) # Minimal SLA if not wallet.is_sla_exempt() and not recipient.is_sla_exempt(): if not wallet.has_valid_sla(): sender_transfers_list = wallet_view_context.authorized_transfers_list( only_appended=False, force_append=True) if len(sender_transfers_list) > settings.SLA_THRESHOLD: raise serializers.ValidationError( detail='', code=ErrorCode.DEBIT_WALLET_EXCEEDED_SLA) elif not recipient.has_valid_sla(): recipient_transfers_list = recipient_view_context.authorized_transfers_list( only_appended=False, force_append=True) if len(recipient_transfers_list) > settings.SLA_THRESHOLD: raise serializers.ValidationError( detail='', code=ErrorCode.CREDIT_WALLET_EXCEEDED_SLA) # Ensure sender log consistency can_append_to_sender_log = wallet_view_context.can_schedule_transfer( ) if can_append_to_sender_log is not True: raise serializers.ValidationError( detail='Sender: {}'.format(can_append_to_sender_log), code=ErrorCode.DEBIT_WALLET_CANNOT_ADD_TRANSACTION) # Ensure recipient log consistency can_append_to_recipient_log = recipient_view_context.can_schedule_transfer( ) if can_append_to_recipient_log is not True: raise serializers.ValidationError( detail='Recipient: {}'.format(can_append_to_recipient_log), code=ErrorCode.CREDIT_WALLET_CANNOT_ADD_TRANSACTION) # Ensure transfer consistency can_spend, currently_available_funds = wallet_view_context.can_send_transfer( current_eon_number=current_eon, using_only_appended_funds=False) if can_spend is not True: raise serializers.ValidationError( detail=can_spend, code=ErrorCode.DEBIT_WALLET_OVERSPENDING) # Validate data concise_balance_marker_signature_data = validated_data.pop( 'debit_balance_signature') concise_balance_marker_amount = validated_data.pop('debit_balance') if concise_balance_marker_amount > currently_available_funds - transfer.amount: raise serializers.ValidationError( detail='', code=ErrorCode.DEBIT_WALLET_BALANCE_MARKER_EXCEED_BALANCE) concise_balance_marker = MinimumAvailableBalanceMarker( wallet=wallet, eon_number=transfer.eon_number, amount=concise_balance_marker_amount) concise_balance_marker_checksum = hex_value( concise_balance_marker.checksum()) concise_balance_marker_signature = Signature( wallet=transfer.wallet, checksum=concise_balance_marker_checksum, value=concise_balance_marker_signature_data.get('value')) if not concise_balance_marker_signature.is_valid(): raise serializers.ValidationError( detail='', code=ErrorCode.INVALID_DEBIT_BALANCE_SIGNATURE) tx_set_tree = wallet_view_context.optimized_authorized_transfers_tree( ) tx_set_hash = hex_value(tx_set_tree.root_hash()) transfer_index = tx_set_tree.merkle_tree_nonce_map.get( transfer.nonce) transfer_proof = tx_set_tree.proof(transfer_index) highest_spendings, highest_gains = wallet_view_context.off_chain_actively_sent_received_amounts( eon_number=transfer.eon_number, only_appended=False) active_state = ActiveState(wallet=wallet, updated_spendings=highest_spendings + transfer.amount, updated_gains=highest_gains, tx_set_hash=tx_set_hash, tx_set_proof_hashes=transfer_proof, tx_set_index=transfer_index, eon_number=transfer.eon_number) checksum = hex_value(active_state.checksum()) active_state_signature = Signature( wallet=transfer.wallet, checksum=checksum, value=active_state_signature_data.get('value')) if not active_state_signature.is_valid(): raise serializers.ValidationError( detail='', code=ErrorCode.INVALID_DEBIT_SIGNATURE) transfer.position = recipient_view_context.off_chain_passively_received_amount( eon_number=transfer.eon_number, only_appended=False) # locking context covers saving the state as well to make sure checkpoint creation is consistent with transaction.atomic(): Signature.objects.bulk_create( [concise_balance_marker_signature, active_state_signature]) concise_balance_marker.signature = concise_balance_marker_signature concise_balance_marker.save() active_state.wallet_signature = active_state_signature active_state.operator_signature = active_state.sign_active_state( settings.HUB_OWNER_ACCOUNT_ADDRESS, settings.HUB_OWNER_ACCOUNT_KEY) active_state.save() transfer.sender_active_state = active_state transfer.sender_balance_marker = concise_balance_marker # cache transfer index in sender active set transfer.sender_merkle_index = transfer_index # transfer.sender_merkle_root_cache = tx_set_hash # cache active set merkle mountains height array and hash array for recipient active set transfer.sender_merkle_hash_cache, transfer.sender_merkle_height_cache = tx_set_tree.merkle_cache_stacks( ) transfer.complete = True transfer.appended = True transfer.processed = True transfer.save() if transfer.appended: operator_celery.send_task('auditor.tasks.on_transfer_confirmation', args=[transfer.id]) return transfer