def create_token_commitment_for_eon(token: Token, eon_number):
logger.info('Creating Token Commitment for {} at {}'.format(
token.address, eon_number))
last_eon_number = eon_number - 1
with transaction.atomic():
wallets = Wallet.objects\
.filter(
token=token,
registration_operator_authorization__isnull=False,
trail_identifier__isnull=False)\
.order_by('trail_identifier')
new_balances = []
left, right = 0, 0
for wallet in wallets:
with wallet.lock(auto_renewal=True):
wallet_transfer_context = WalletTransferContext(
wallet=wallet, transfer=None)
last_transfer, last_transfer_is_outgoing = wallet_transfer_context.last_appended_active_transfer(
eon_number=last_eon_number)
last_transfer_active_state = None
if last_transfer is not None and last_transfer.is_open_swap():
last_transfer.retire_swap()
if last_transfer is not None:
last_transfer_active_state = WalletTransferContext.appropriate_transfer_active_state(
transfer=last_transfer,
is_outgoing=last_transfer_is_outgoing)
available_funds = wallet_transfer_context.available_funds_at_eon(
eon_number=last_eon_number,
only_appended=True)
right = left + available_funds
assert right >= left, 'Wallet {} Token {} Balance {}'.format(
wallet.address, token.address, available_funds)
passive_checksum, passive_amount, passive_marker = wallet_transfer_context.get_passive_values(
eon_number=last_eon_number)
new_balances.append({
'contract': settings.HUB_LQD_CONTRACT_ADDRESS,
'token': token.address,
'wallet': wallet.address,
'left': left,
'right': right,
'active_state_checksum': last_transfer_active_state.checksum() if last_transfer_active_state is not None else b'\0'*32,
'active_state': last_transfer_active_state,
'passive_checksum': passive_checksum,
'passive_amount': passive_amount,
'passive_marker': passive_marker,
})
left = right
last_incoming_passive_transfer = wallet_transfer_context.last_appended_incoming_passive_transfer(
eon_number=last_eon_number)
if last_incoming_passive_transfer:
wallet_transfer_context = WalletTransferContext(
wallet=wallet, transfer=last_incoming_passive_transfer)
passive_eon_transfers_list = wallet_transfer_context.incoming_passive_transfers_list(
only_appended=True,
force_append=False)
passive_transfers_merkle_tree = wallet_transfer_context.incoming_passive_transfers_tree(
only_appended=True,
force_append=False)
for index, incoming_passive_transfer in enumerate(passive_eon_transfers_list):
final_transfer_index = index
final_transfer_membership_proof = passive_transfers_merkle_tree.proof(
final_transfer_index)
final_transfer_membership_proof_chain = final_transfer_membership_proof.get(
"chain")
final_transfer_membership_proof_values = final_transfer_membership_proof.get(
"values")
assert incoming_passive_transfer.final_receipt_hashes is None
assert incoming_passive_transfer.final_receipt_index is None
assert incoming_passive_transfer.final_receipt_values is None
incoming_passive_transfer.final_receipt_hashes = final_transfer_membership_proof_chain
incoming_passive_transfer.final_receipt_index = final_transfer_index
incoming_passive_transfer.final_receipt_values = final_transfer_membership_proof_values
incoming_passive_transfer.save()
if last_transfer_active_state is None:
continue
wallet_transfer_context = WalletTransferContext(
wallet=wallet, transfer=last_transfer)
starting_balance = int(
wallet_transfer_context.starting_balance_in_eon(last_eon_number))
# if last active transfer is a multi eon swap
# starting balance included in every tx checksum should be set to the cached starting balance
# this way checkpoint state will match signed active state
if last_transfer.is_swap() and not last_transfer.cancelled:
if Transfer.objects.filter(eon_number=last_transfer.eon_number-1, tx_id=last_transfer.tx_id).exists():
matched_out, matched_in = last_transfer.matched_amounts(
all_eons=True)
current_matched_out, current_matched_in = last_transfer.matched_amounts(
all_eons=False)
if last_transfer_is_outgoing:
sender_starting_balance = last_transfer.sender_starting_balance
# current eon's starting balance should be equal to
# cached starting balance - committed matched out amount in past rounds
assert(starting_balance == sender_starting_balance -
matched_out + current_matched_out)
starting_balance = sender_starting_balance
else:
recipient_starting_balance = last_transfer.recipient_starting_balance
# current eon's starting balance should be equal to
# cached starting balance + committed matched in amount in past rounds
assert(
starting_balance == recipient_starting_balance + matched_in - current_matched_in)
starting_balance = recipient_starting_balance
confirmed_eon_transfers_list = wallet_transfer_context.authorized_transfers_list(
only_appended=True,
force_append=False)
confirmed_eon_transfers_list_shorthand = wallet_transfer_context.authorized_transfers_list_shorthand(
only_appended=True,
force_append=False,
last_transfer_is_finalized=False,
starting_balance=starting_balance)
transaction_merkle_tree = TransactionMerkleTree(
confirmed_eon_transfers_list_shorthand)
transaction_merkle_tree_root = hex_value(
transaction_merkle_tree.root_hash())
assert transaction_merkle_tree_root == last_transfer_active_state.tx_set_hash,\
'{}/{}'.format(transaction_merkle_tree_root,
last_transfer_active_state.tx_set_hash)
for confirmed_incoming_transfer in confirmed_eon_transfers_list:
if confirmed_incoming_transfer.recipient != wallet:
continue
final_transfer_index = transaction_merkle_tree.merkle_tree_nonce_map.get(
confirmed_incoming_transfer.nonce)
final_transfer_membership_proof_chain = transaction_merkle_tree.proof(
final_transfer_index)
assert confirmed_incoming_transfer.final_receipt_hashes is None
assert confirmed_incoming_transfer.final_receipt_index is None
confirmed_incoming_transfer.final_receipt_hashes = final_transfer_membership_proof_chain
confirmed_incoming_transfer.final_receipt_index = final_transfer_index
confirmed_incoming_transfer.save()
managed_funds = 0
if eon_number > 1:
last_eon = LocalViewInterface.confirmed(eon_number=last_eon_number)
pending_withdrawals_until_last_eon = \
WithdrawalRequest.objects\
.filter(wallet__token=token, eon_number__lte=last_eon_number, slashed=False)\
.filter(Q(withdrawal__isnull=True) | Q(withdrawal__block__gt=last_eon.block))
if not pending_withdrawals_until_last_eon.exists():
last_eon_pending_withdrawals = 0
else:
last_eon_pending_withdrawals = pending_withdrawals_until_last_eon\
.aggregate(Sum('amount')) \
.get('amount__sum')
total_token_balance = last_eon.contractledgerstate_set.get(
token=token).total_balance
managed_funds = total_token_balance - last_eon_pending_withdrawals
if right < managed_funds:
logger.warning('UNCLAIMED FUNDS: {} in {}'.format(
managed_funds - right, token.address))
send_admin_email(
subject='Soft TokenCommitment Warning: Extra funds',
content='There are some additional funds in the balance pool that belong to no one: {} of {}'
.format(managed_funds - right, token.address))
altered_balances = new_balances + [{
'contract': settings.HUB_LQD_CONTRACT_ADDRESS,
'token': token.address,
'wallet': settings.HUB_OWNER_ACCOUNT_ADDRESS,
'left': left,
'right': managed_funds,
'active_state_checksum': b'\0'*32,
'active_state': None,
'passive_checksum': b'\0'*32,
'passive_amount': 0,
'passive_marker': 0,
}]
new_merkle_tree = MerkleTree(altered_balances, managed_funds)
right = managed_funds
else:
if right > managed_funds:
logger.error('OVERCLAIMING FUNDS!! {} > {} in {}'.format(
right, managed_funds, token.address))
send_admin_email(
subject='HARD Checkpoint Error: OVERCLAIMING!',
content='OVERCLAIMING FUNDS!! {} > {} in {}'.format(right, managed_funds, token.address))
new_merkle_tree = MerkleTree(new_balances, right)
bulk_manager = BulkCreateManager(chunk_size=500)
for index, balance in enumerate(new_balances):
if not balance.get('wallet') or balance.get('wallet') == '0x0000000000000000000000000000000000000000':
continue
merkle_proof = new_merkle_tree.proof(index)
wallet = Wallet.objects.get(
token=token,
address=remove_0x_prefix(balance.get('wallet')))
# TODO verify validity through RPC prior to insertion
assert(wallet.trail_identifier == index)
# create records in batches
bulk_manager.add(
ExclusiveBalanceAllotment(
wallet=wallet,
eon_number=eon_number,
left=balance.get('left'),
right=balance.get('right'),
merkle_proof_hashes=merkle_proof.get('chain'),
merkle_proof_values=merkle_proof.get('values'),
merkle_proof_trail=index,
active_state=balance.get('active_state')
)
)
# make sure remaining batch is added
bulk_manager.done()
token_commitment = TokenCommitment.objects.create(
token=token,
merkle_root=hex_value(new_merkle_tree.root_hash()),
upper_bound=right)
return token_commitment
def create(self, validated_data):
active_state_signature_data = validated_data.pop('debit_signature')
wallet = validated_data.pop('wallet')
recipient = validated_data.pop('recipient')
# get current eon
current_eon = LocalViewInterface.latest().eon_number()
# transfer eon should be the current eon number
if validated_data.pop('eon_number') != current_eon:
raise serializers.ValidationError(
detail='', code=ErrorCode.EON_NUMBER_OUT_OF_SYNC)
# TODO refactor this such that the recipient is only locked after the sender's details are verified
wallets = sorted([wallet, recipient], key=lambda w: w.trail_identifier)
with RootCommitment.read_write_lock(
suffix=current_eon, auto_renewal=False), wallets[0].lock(
auto_renewal=False), wallets[1].lock(auto_renewal=False):
if RootCommitment.objects.filter(eon_number=current_eon +
1).exists():
raise serializers.ValidationError(
detail='', code=ErrorCode.EON_NUMBER_OUT_OF_SYNC)
transfer = Transfer(wallet=wallet,
amount=validated_data.pop('amount'),
eon_number=current_eon,
recipient=recipient,
nonce=validated_data.pop('nonce'),
passive=True)
wallet_view_context = WalletTransferContext(wallet=wallet,
transfer=transfer)
recipient_view_context = WalletTransferContext(wallet=recipient,
transfer=transfer)
# Minimal SLA
if not wallet.is_sla_exempt() and not recipient.is_sla_exempt():
if not wallet.has_valid_sla():
sender_transfers_list = wallet_view_context.authorized_transfers_list(
only_appended=False, force_append=True)
if len(sender_transfers_list) > settings.SLA_THRESHOLD:
raise serializers.ValidationError(
detail='',
code=ErrorCode.DEBIT_WALLET_EXCEEDED_SLA)
elif not recipient.has_valid_sla():
recipient_transfers_list = recipient_view_context.authorized_transfers_list(
only_appended=False, force_append=True)
if len(recipient_transfers_list) > settings.SLA_THRESHOLD:
raise serializers.ValidationError(
detail='',
code=ErrorCode.CREDIT_WALLET_EXCEEDED_SLA)
# Ensure sender log consistency
can_append_to_sender_log = wallet_view_context.can_schedule_transfer(
)
if can_append_to_sender_log is not True:
raise serializers.ValidationError(
detail='Sender: {}'.format(can_append_to_sender_log),
code=ErrorCode.DEBIT_WALLET_CANNOT_ADD_TRANSACTION)
# Ensure recipient log consistency
can_append_to_recipient_log = recipient_view_context.can_schedule_transfer(
)
if can_append_to_recipient_log is not True:
raise serializers.ValidationError(
detail='Recipient: {}'.format(can_append_to_recipient_log),
code=ErrorCode.CREDIT_WALLET_CANNOT_ADD_TRANSACTION)
# Ensure transfer consistency
can_spend, currently_available_funds = wallet_view_context.can_send_transfer(
current_eon_number=current_eon,
using_only_appended_funds=False)
if can_spend is not True:
raise serializers.ValidationError(
detail=can_spend, code=ErrorCode.DEBIT_WALLET_OVERSPENDING)
# Validate data
concise_balance_marker_signature_data = validated_data.pop(
'debit_balance_signature')
concise_balance_marker_amount = validated_data.pop('debit_balance')
if concise_balance_marker_amount > currently_available_funds - transfer.amount:
raise serializers.ValidationError(
detail='',
code=ErrorCode.DEBIT_WALLET_BALANCE_MARKER_EXCEED_BALANCE)
concise_balance_marker = MinimumAvailableBalanceMarker(
wallet=wallet,
eon_number=transfer.eon_number,
amount=concise_balance_marker_amount)
concise_balance_marker_checksum = hex_value(
concise_balance_marker.checksum())
concise_balance_marker_signature = Signature(
wallet=transfer.wallet,
checksum=concise_balance_marker_checksum,
value=concise_balance_marker_signature_data.get('value'))
if not concise_balance_marker_signature.is_valid():
raise serializers.ValidationError(
detail='', code=ErrorCode.INVALID_DEBIT_BALANCE_SIGNATURE)
tx_set_tree = wallet_view_context.optimized_authorized_transfers_tree(
)
tx_set_hash = hex_value(tx_set_tree.root_hash())
transfer_index = tx_set_tree.merkle_tree_nonce_map.get(
transfer.nonce)
transfer_proof = tx_set_tree.proof(transfer_index)
highest_spendings, highest_gains = wallet_view_context.off_chain_actively_sent_received_amounts(
eon_number=transfer.eon_number, only_appended=False)
active_state = ActiveState(wallet=wallet,
updated_spendings=highest_spendings +
transfer.amount,
updated_gains=highest_gains,
tx_set_hash=tx_set_hash,
tx_set_proof_hashes=transfer_proof,
tx_set_index=transfer_index,
eon_number=transfer.eon_number)
checksum = hex_value(active_state.checksum())
active_state_signature = Signature(
wallet=transfer.wallet,
checksum=checksum,
value=active_state_signature_data.get('value'))
if not active_state_signature.is_valid():
raise serializers.ValidationError(
detail='', code=ErrorCode.INVALID_DEBIT_SIGNATURE)
transfer.position = recipient_view_context.off_chain_passively_received_amount(
eon_number=transfer.eon_number, only_appended=False)
# locking context covers saving the state as well to make sure checkpoint creation is consistent
with transaction.atomic():
Signature.objects.bulk_create(
[concise_balance_marker_signature, active_state_signature])
concise_balance_marker.signature = concise_balance_marker_signature
concise_balance_marker.save()
active_state.wallet_signature = active_state_signature
active_state.operator_signature = active_state.sign_active_state(
settings.HUB_OWNER_ACCOUNT_ADDRESS,
settings.HUB_OWNER_ACCOUNT_KEY)
active_state.save()
transfer.sender_active_state = active_state
transfer.sender_balance_marker = concise_balance_marker
# cache transfer index in sender active set
transfer.sender_merkle_index = transfer_index
# transfer.sender_merkle_root_cache = tx_set_hash
# cache active set merkle mountains height array and hash array for recipient active set
transfer.sender_merkle_hash_cache, transfer.sender_merkle_height_cache = tx_set_tree.merkle_cache_stacks(
)
transfer.complete = True
transfer.appended = True
transfer.processed = True
transfer.save()
if transfer.appended:
operator_celery.send_task('auditor.tasks.on_transfer_confirmation',
args=[transfer.id])
return transfer