def register_owner_account(token: Token):
if Wallet.objects.filter(token=token, address__iexact=remove_0x_prefix(settings.HUB_OWNER_ACCOUNT_ADDRESS)).exists():
logger.error('Owner account already registered.')
return
if not LocalViewInterface.get_contract_parameters():
logger.error('Contract parameters not yet populated.')
return
logger.warning('Registering owner account: {}'.format(
settings.HUB_OWNER_ACCOUNT_ADDRESS))
latest_eon_number = LocalViewInterface.latest().eon_number()
authorization_digest = Wallet(
token=token, address=settings.HUB_OWNER_ACCOUNT_ADDRESS).get_admission_hash(latest_eon_number)
authorization = sign_message(
authorization_digest, settings.HUB_OWNER_ACCOUNT_KEY)
latest_tos_config = TOSConfig.objects.all().order_by('time').last()
tos_signature = sign_message(
hex_value(latest_tos_config.digest()), settings.HUB_OWNER_ACCOUNT_KEY)
registration = AdmissionRequestSerializer(data={
'token': token.address,
'address': remove_0x_prefix(settings.HUB_OWNER_ACCOUNT_ADDRESS),
'authorization': {
'value': encode_signature(authorization)
},
'tos_signature': {
'value': encode_signature(tos_signature)
}
})
registration.is_valid(raise_exception=True)
registration.save()
process_admissions()
def register_owner_account(token: Token):
if Wallet.objects.filter(token=token,
address=remove_0x_prefix(
settings.HUB_OWNER_ACCOUNT_ADDRESS)).exists():
return
if not LocalViewInterface.get_contract_parameters():
return
latest_eon_number = LocalViewInterface.latest().eon_number()
authorization_digest = Wallet(
token=token,
address=settings.HUB_OWNER_ACCOUNT_ADDRESS).get_admission_hash(
latest_eon_number)
authorization = sign_message(authorization_digest,
settings.HUB_OWNER_ACCOUNT_KEY)
latest_tos_config = TOSConfig.objects.all().order_by('time').last()
tos_signature = sign_message(decode_hex(latest_tos_config.digest()),
settings.HUB_OWNER_ACCOUNT_KEY)
registration = AdmissionRequestSerializer(
data={
'token': token.address,
'address': remove_0x_prefix(settings.HUB_OWNER_ACCOUNT_ADDRESS),
'authorization': {
'value': encode_signature(authorization)
},
'tos_signature': {
'value': encode_signature(tos_signature)
}
})
registration.is_valid(raise_exception=True)
registration.save()
def get_wallet_data(channel_name,
operation,
wallet_address,
token_address,
eon_number=None):
# default eon_number is the latest eon
if eon_number is None:
eon_number = LocalViewInterface.latest().eon_number()
# wrap negative eon_number
if eon_number < 0:
eon_number += LocalViewInterface.latest().eon_number()
try:
wallet = Wallet.objects.get(address__iexact=wallet_address,
token__address__iexact=token_address)
request_model = MockModel(eon_number=eon_number,
wallet=wallet,
transfer_id=0)
data = WalletStateSerializer(request_model).data
# send response to websocket channel
async_to_sync(send_response)(
channel_name=channel_name,
resource="wallet",
data=data,
)
except Wallet.DoesNotExist:
# send error to websocket channel
async_to_sync(send_error)(channel_name=channel_name,
error='Wallet does not exist.',
cause=operation)
def cancel_finalize_swaps():
if not LocalViewInterface.get_contract_parameters():
logger.error('Contract parameters not yet populated.')
return
latest_eon_number = LocalViewInterface.latest().eon_number()
# This lock is required because the ledger will be mutated as the swaps are processed
with RootCommitment.global_lock():
cancel_finalize_swaps_for_eon(latest_eon_number)
def process_passive_transfers():
logger.info('Processing passive transfers')
if not LocalViewInterface.get_contract_parameters():
logger.error('Contract parameters not yet populated.')
return
latest_eon_number = LocalViewInterface.latest().eon_number()
# This lock is required because the ledger will be mutated as the transfers are processed
with RootCommitment.global_lock():
logger.info('Start')
process_passive_transfers_for_eon(latest_eon_number)
def create_checkpoint_for_eon(eon_number, latest_block_number):
if RootCommitment.objects.filter(eon_number=eon_number).count() > 0:
return False
if eon_number > 1:
last_eon_number = eon_number - 1
last_eon = LocalViewInterface.confirmed(eon_number=last_eon_number)
if not last_eon:
logger.error(
'Missing confirmed contract state for eon {}.'.format(last_eon_number))
send_admin_email(
subject='Soft Checkpoint Error: Missing Contract State',
content='Missing confirmed contract state for previous eon {}. We may not be in sync with the blockchain!'.format(last_eon_number))
return False
last_confirmed_eon_number, last_confirmed_sub_block = last_eon.eon_number_and_sub_block()
if last_confirmed_eon_number != last_eon_number:
logger.error(
'Attempted to use confirmed state for eon {}. Expected {}.'.format(last_confirmed_eon_number,
last_eon_number))
send_admin_email(
subject='Soft Checkpoint Error: Wrong last eon #',
content='Need to sync chain! Attempted to use confirmed state for eon {}. Expected {}!'
.format(last_confirmed_eon_number,
last_eon_number))
return False
last_sub_block_number = LocalViewInterface.get_contract_parameters().blocks_per_eon - 1
if last_confirmed_sub_block != last_sub_block_number:
logger.error(
'Attempted to use confirmed state for sub block {}. Expected {}.'.format(last_confirmed_sub_block,
last_sub_block_number))
send_admin_email(
subject='Soft Checkpoint Error: Wrong last Sub block #',
content='Need to sync chain! Attempted to use confirmed state for sub block {}. Expected {}.'
.format(last_confirmed_sub_block,
last_sub_block_number))
return False
# commitment write read lock makes sure transaction confirmation will not mutate ledger while checkpoint is being created
with transaction.atomic(), RootCommitment.read_write_lock(suffix=eon_number-1, is_write=True, auto_renewal=True):
# TODO parallelism
token_commitments = [create_token_commitment_for_eon(token, eon_number) for token in
Token.objects.all().order_by('trail')]
root_commitment = create_root_commitment_for_eon(
token_commitments, eon_number, latest_block_number)
NOCUSTContractInterface().queue_submit_checkpoint(root_commitment)
return True
def register_token(token_address, name, short_name, register_on_chain):
try:
Token.objects.get(address=remove_0x_prefix(token_address))
raise ValueError(
'Token {} already registered in local db.'.format(token_address))
except Token.DoesNotExist:
pass
if register_on_chain:
NOCUSTContractInterface().register_ERC20(token_address)
print('Registration transaction queued.')
token = Token.objects.create(address=remove_0x_prefix(token_address),
name=name,
short_name=short_name,
trail=Token.objects.count(),
block=LocalViewInterface.latest_block())
print('Token locally registered.')
register_owner_account(token)
if same_hex_value(token_address, settings.SLA_TOKEN_ADDRESS):
register_sla_recipient_account()
return token
def create_root_commitment_for_eon(token_commitments: [TokenCommitment], eon_number, latest_block_number):
token_commitment_leaves = [commitment.shorthand()
for commitment in token_commitments]
token_merkle_tree = TokenMerkleTree(token_commitment_leaves)
token_merkle_tree_root = hex_value(token_merkle_tree.root_hash())
previous_eon_basis = ZERO_CHECKSUM
if eon_number > 1:
local_block = LocalViewInterface.confirmed(eon_number - 1)
print(local_block.__dict__)
print(local_block.eon_number())
previous_eon_basis = local_block.basis
root_commitment = RootCommitment.objects.create(
eon_number=eon_number,
basis=previous_eon_basis,
merkle_root=token_merkle_tree_root,
block=latest_block_number)
for token_commitment in token_commitments:
token_commitment.root_commitment = root_commitment
token_commitment.membership_hashes = token_merkle_tree.proof(
token_commitment.token.trail)
token_commitment.save()
return root_commitment
def get_last_checkpoint(self, block_identifier='latest'):
eon_number = self.get_last_checkpoint_submission_eon()
eons_kept = LocalViewInterface.get_contract_parameters().eons_kept
return self.contract\
.functions\
.getCheckpointAtSlot(eon_number % eons_kept)\
.call(block_identifier=block_identifier)
def broadcast_checkpoint():
"""
References to this function is absent in program code
"""
with ContractState.global_lock():
latest_block = LocalViewInterface.latest()
submitted = latest_block.is_checkpoint_submitted_for_current_eon
current_eon, current_sub_block = latest_block.eon_number_and_sub_block()
blocks_for_submission = LocalViewInterface.blocks_for_submission()
if submitted:
logger.warning("TokenCommitment already submitted")
return
elif current_sub_block < blocks_for_submission:
logger.warning('Too early to submit checkpoint: {} blocks left'.format(
blocks_for_submission - current_sub_block))
return
elif current_sub_block > 150 and settings.DEBUG:
logger.error("just let the damn tests pass..") # TODO: todo
return
elif latest_block.has_missed_checkpoint_submission:
logger.error(
'The operator has missed a checkpoint submission. Cannot submit checkpoint.')
send_admin_email(
subject='The commit chain is halted.',
content='Ouch.')
return
checkpoint = TokenCommitment.objects.get(eon_number=current_eon)
if EthereumTransaction.objects.filter(tag=checkpoint.tag()).exists():
logger.warning("TokenCommitment already enqueued.")
send_admin_email(
subject='Soft Submission Error: TokenCommitment already enqueued.',
content='This should eventually be resolved.')
return
managed_funds = NOCUSTContractInterface().get_managed_funds(checkpoint.eon_number)
if checkpoint.upper_bound > managed_funds:
logger.error(
"TokenCommitment upper bound greater than managed funds.")
send_admin_email(
subject='HARD Submission Error: TokenCommitment upper bound greater than managed funds.',
content='Created checkpoint for {} while managed funds are {}. Some withdrawals are possibly pending cancellation.'.format(checkpoint.upper_bound, managed_funds))
return
NOCUSTContractInterface().queue_submit_checkpoint(checkpoint)
def process_admissions():
if not LocalViewInterface.get_contract_parameters():
logger.error('Contract parameters not yet populated.')
return
# This lock is needed because new wallets can be introduced, which would affect the checkpoint.
with RootCommitment.global_lock():
process_admissions_for_latest_eon()
def get_deposits(self,
token_address,
eon_number,
block_identifier='latest'):
eons_kept = LocalViewInterface.get_contract_parameters().eons_kept
aggregate_eon, aggregate_value = self.contract\
.functions\
.getDepositsAtSlot(add_0x_prefix(token_address), eon_number % eons_kept)\
.call(block_identifier=block_identifier)
return aggregate_value if aggregate_eon == eon_number else 0
def get_operator_data(channel_name, operation):
latest = LocalViewInterface.latest()
confirmed = LocalViewInterface.confirmed()
data = {
'latest': {
'block': latest.block,
'eon_number': latest.eon_number(),
},
'confirmed': {
'block': confirmed.block,
'eon_number': confirmed.eon_number(),
}
}
# send response to websocket channel
async_to_sync(send_response)(
channel_name=channel_name,
resource="operator",
data=data,
)
def create_checkpoint():
if not LocalViewInterface.get_contract_parameters():
logger.error('Contract parameters not yet populated.')
return
latest = LocalViewInterface.latest()
latest_eon_number, latest_sub_block = latest.eon_number_and_sub_block()
blocks_for_creation = LocalViewInterface.blocks_for_creation()
confirmed_eon_number, confirmed_sub_block = LocalViewInterface.confirmed(
).eon_number_and_sub_block()
if confirmed_eon_number < latest_eon_number:
return
if latest_sub_block < blocks_for_creation:
return
with RootCommitment.global_lock():
new_checkpoint = create_checkpoint_for_eon(latest_eon_number, latest.block)
if new_checkpoint:
operator_celery.send_task('auditor.tasks.broadcast_wallet_data')
def register_sla_recipient_account():
if same_hex_value(settings.SLA_RECIPIENT_ADDRESS,
settings.HUB_OWNER_ACCOUNT_ADDRESS):
logger.warning('Skipping registration: Hub Owner is SLA recipient.')
return
token = Token.objects.filter(
address__iexact=remove_0x_prefix(settings.SLA_TOKEN_ADDRESS))
if not token.exists():
logger.error('SLA Payment Token not yet registered.')
return
if Wallet.objects.filter(token=token,
address__iexact=remove_0x_prefix(
settings.SLA_RECIPIENT_ADDRESS)).exists():
logger.error('Recipient account already registered.')
return
if not LocalViewInterface.get_contract_parameters():
logger.error('Contract parameters not yet populated.')
return
latest_eon = LocalViewInterface.latest().eon_number()
authorization_digest = Wallet(
token=token,
address=settings.SLA_RECIPIENT_ADDRESS).get_admission_hash(latest_eon)
authorization = sign_message(authorization_digest,
settings.SLA_RECIPIENT_KEY)
registration = AdmissionRequestSerializer(
data={
'token': token.address,
'address': remove_0x_prefix(settings.SLA_RECIPIENT_ADDRESS),
'authorization': encode_signature(authorization)
})
registration.is_valid(raise_exception=True)
registration.save()
process_admissions()
def cache_wallet_data(eon_number, token_address, wallet_address, data):
if eon_number < LocalViewInterface.latest().eon_number():
path = f"/audit_data_cache/{eon_number}/{token_address}"
if not os.path.exists(path):
os.makedirs(path)
with open(f"{path}/{wallet_address}.json", "w+") as f:
f.write(json.dumps(data))
logger.info(f"Cached {eon_number}/{token_address}/{wallet_address} .")
else:
logger.info(
f"Skipping cache for {eon_number}/{token_address}/{wallet_address}, eon {eon_number} is not over yet."
)
def register_eth_token():
if Token.objects.filter(address__iexact=remove_0x_prefix(
settings.HUB_LQD_CONTRACT_ADDRESS)).exists():
logger.error('ETH token already registered.')
return
if not LocalViewInterface.get_contract_parameters():
logger.error('Contract parameters not yet populated.')
return
logger.warning('Registering ETH Token')
eth_token = register_token(token_address=settings.HUB_LQD_CONTRACT_ADDRESS,
name='Ethereum',
short_name='ETH',
register_on_chain=False)
ContractLedgerState.objects.create(
contract_state=LocalViewInterface.genesis(),
token=eth_token,
pending_withdrawals=0,
confirmed_withdrawals=0,
deposits=0,
total_balance=0)
def broadcast_wallet_data():
eon_number = LocalViewInterface.latest().eon_number()
for wallet in Wallet.objects.all():
request_model = MockModel(eon_number=eon_number,
wallet=wallet,
transfer_id=0)
data = WalletStateSerializer(request_model).data
send_notification(stream_prefix='wallet',
stream_id="{}/{}".format(wallet.token.address,
wallet.address),
event_name=CHECKPOINT_CREATED,
data=data)
def synchronize_contract_state(verbose=False):
logger.info('Retrieving confirmed events')
if not LocalViewInterface.get_contract_parameters():
logger.error('Contract parameters not yet populated.')
return
with ContractState.global_lock():
logger.info('Start..')
contract_interface = NOCUSTContractInterface()
logger.info('Interface acquired')
contract_event_decoder = NOCUSTContractEventDecoder()
logger.info('Decoder acquired')
contract_interface.get_blocks_per_eon()
return concurrently_retrieve_state(contract_interface,
contract_event_decoder, verbose)
def validate(self, attrs):
authorization = attrs.pop('authorization').get('value')
address = attrs.get('address')
token = attrs.get('token')
tos_signature = attrs.pop('tos_signature').get('value')
if BlacklistEntry.objects.filter(
address__iexact=remove_0x_prefix(address)).exists():
raise serializers.ValidationError(
detail='', code=ErrorCode.WALLET_BLACKLISTED)
if Wallet.objects.filter(address__iexact=remove_0x_prefix(address),
token=token).exists():
raise serializers.ValidationError(
detail='', code=ErrorCode.WALLET_ALREADY_ADMITTED)
dummy_wallet = Wallet(token=token, address=remove_0x_prefix(address))
attrs['registration_eon_number'] = LocalViewInterface.latest(
).eon_number()
admission_hash = hex_value(
dummy_wallet.get_admission_hash(attrs['registration_eon_number']))
attrs['signature'] = Signature(wallet=dummy_wallet,
checksum=admission_hash,
value=authorization)
if not attrs['signature'].is_valid():
raise serializers.ValidationError(
detail='', code=ErrorCode.INVALID_ADMISSION_SIGNATURE)
latest_tos_config = TOSConfig.objects.all().order_by('time').last()
attrs['tos_signature'] = Signature(wallet=dummy_wallet,
checksum=latest_tos_config.digest(),
value=tos_signature)
if not attrs['tos_signature'].is_valid():
raise serializers.ValidationError(
detail='Invalid TOS (digest: {}) signature'.format(
latest_tos_config.digest()),
code=ErrorCode.INVALID_TOS_SIGNATURE)
attrs['tos_config'] = latest_tos_config
return attrs
def update(self, swap, validated_data):
with transaction.atomic():
current_eon = LocalViewInterface.latest().eon_number()
swap_set = Transfer.objects.select_for_update().filter(
tx_id=swap.tx_id, eon_number__gte=current_eon,
swap=True).order_by('eon_number')
current_swap = swap_set[0]
if current_swap.complete:
raise serializers.ValidationError(
detail='', code=ErrorCode.SWAP_ALREADY_FULFILLED)
elif current_swap.cancelled:
raise serializers.ValidationError(
detail='', code=ErrorCode.SWAP_ALREADY_FROZEN)
elif current_swap.voided:
raise serializers.ValidationError(
detail='', code=ErrorCode.SWAP_ALREADY_VOIDED)
elif current_swap.processed:
raise serializers.ValidationError(
detail='', code=ErrorCode.SWAP_ALREADY_CLOSED)
freezing_signature_data = validated_data.pop('freezing_signature')
freezing_checksum = crypto.hex_value(
current_swap.swap_cancellation_message_checksum())
freezing_signature = Signature(
wallet=current_swap.wallet,
checksum=freezing_checksum,
value=freezing_signature_data.get('value'))
if not freezing_signature.is_valid():
raise serializers.ValidationError(
detail='', code=ErrorCode.INVALID_FREEZING_SIGNATURE)
freezing_signature.save()
# only current swap should be locked, future swaps are not matched
with current_swap.lock(
auto_renewal=False), current_swap.wallet.lock(
auto_renewal=False), current_swap.recipient.lock(
auto_renewal=False):
swap_set.update(cancelled=True,
swap_freezing_signature=freezing_signature)
return current_swap
def to_representation(self, swap_data_request: SwapDataRequest):
latest = LocalViewInterface.latest().eon_number()
eon_number = swap_data_request.eon_number if 0 <= swap_data_request.eon_number <= latest else latest
all_sell_orders = Transfer.objects.filter(
wallet__token=swap_data_request.left_token,
recipient__token=swap_data_request.right_token,
processed=False,
complete=False,
voided=False,
cancelled=False,
swap=True,
eon_number=eon_number)
all_sell_orders = sorted(all_sell_orders,
key=cmp_to_key(
price_comparison_function(inverse=True)))
all_buy_orders = Transfer.objects.filter(
wallet__token=swap_data_request.right_token,
recipient__token=swap_data_request.left_token,
processed=False,
complete=False,
voided=False,
cancelled=False,
swap=True,
eon_number=eon_number)
all_buy_orders = sorted(all_buy_orders,
key=cmp_to_key(
price_comparison_function(inverse=False)))
return {
'sell_orders':
OrderSerializer(combine_order_volumes(all_sell_orders),
many=True,
read_only=True).data,
'buy_orders':
OrderSerializer(combine_order_volumes(all_buy_orders),
many=True,
read_only=True).data,
}
def fetch_contract_state_at_block(self, block_number):
try:
local_params = LocalViewInterface.get_contract_parameters()
current_eon = 1 + \
(block_number - local_params.genesis_block) // local_params.blocks_per_eon
contract_state_variables = self.contract\
.functions\
.getServerContractStateVariables()\
.call(block_identifier=block_number)
basis = contract_state_variables[0]
last_checkpoint_submission_eon = contract_state_variables[1]
last_checkpoint = contract_state_variables[2]
is_checkpoint_submitted_for_current_eon = contract_state_variables[
3]
has_missed_checkpoint_submission = contract_state_variables[4]
live_challenge_count = contract_state_variables[5]
except Exception as exception:
traceback.print_exc()
logger.error('Could not query contract state: {}'.format(
str(exception)))
return None
contract_state = ContractState(
block=block_number,
confirmed=False,
basis=crypto.hex_value(basis),
last_checkpoint_submission_eon=last_checkpoint_submission_eon,
last_checkpoint=crypto.hex_value(last_checkpoint),
is_checkpoint_submitted_for_current_eon=
is_checkpoint_submitted_for_current_eon,
has_missed_checkpoint_submission=has_missed_checkpoint_submission,
live_challenge_count=live_challenge_count)
contract_ledger_states = []
for token in Token.objects.all():
if token.block >= block_number:
continue
try:
contract_state_ledger_variables = self.contract\
.functions\
.getServerContractLedgerStateVariables(current_eon, add_0x_prefix(token.address))\
.call(block_identifier=block_number)
pending_withdrawals = contract_state_ledger_variables[0]
confirmed_withdrawals = contract_state_ledger_variables[1]
deposits = contract_state_ledger_variables[2]
total_balance = contract_state_ledger_variables[3]
contract_ledger_states.append(
ContractLedgerState(
token=token,
pending_withdrawals=pending_withdrawals,
confirmed_withdrawals=confirmed_withdrawals,
deposits=deposits,
total_balance=total_balance))
except Exception as exception:
traceback.print_exc()
logger.error(
'Could not query contract ledger state for {}: {}'.format(
token.address, str(exception)))
contract_ledger_states.append(
ContractLedgerState(token=token,
pending_withdrawals=0,
confirmed_withdrawals=0,
deposits=0,
total_balance=0))
return contract_state, contract_ledger_states
def get_basis(self, eon_number, block_identifier='latest'):
eons_kept = LocalViewInterface.get_contract_parameters().eons_kept
return self.contract\
.functions\
.getParentChainAccumulatorAtSlot(eon_number % eons_kept)\
.call(block_identifier=block_identifier)
def update(self, swap, validated_data):
current_swap = None
is_swap_finalized = False
with transaction.atomic():
current_eon = LocalViewInterface.latest().eon_number()
swap_set = Transfer.objects.select_for_update().filter(
tx_id=swap.tx_id, eon_number__gte=current_eon,
swap=True).order_by('eon_number')
current_swap = swap_set[0]
if not current_swap.complete:
raise serializers.ValidationError(
detail='', code=ErrorCode.SWAP_NOT_FULFILLED)
elif current_swap.cancelled:
raise serializers.ValidationError(
detail='', code=ErrorCode.SWAP_ALREADY_FROZEN)
elif current_swap.voided:
raise serializers.ValidationError(
detail='', code=ErrorCode.SWAP_ALREADY_VOIDED)
elif current_swap.processed:
raise serializers.ValidationError(
detail='', code=ErrorCode.SWAP_ALREADY_CLOSED)
elif current_swap.recipient_finalization_active_state is not None:
raise serializers.ValidationError(
detail='', code=ErrorCode.SWAP_ALREADY_FINALIZED)
finalization_signatures = validated_data.pop(
'finalization_signature')
# state to save
finalization_active_state_signature_records = []
finalization_active_state_records = []
if swap_set.count() != len(finalization_signatures):
raise serializers.ValidationError(
detail=
'Wrong number of finalization signatures, expected {} but got {}'
.format(swap_set.count(), len(finalization_signatures)),
code=ErrorCode.WRONG_NUMBER_OF_CREDIT_SIGNATURES)
recipient_view_context = WalletTransferContext(
wallet=current_swap.recipient, transfer=current_swap)
tx_set_tree = recipient_view_context.optimized_authorized_transfers_tree(
)
tx_set_hash = crypto.hex_value(tx_set_tree.root_hash())
transfer_index = tx_set_tree.merkle_tree_nonce_map.get(
current_swap.nonce)
transfer_proof = tx_set_tree.proof(transfer_index)
highest_spendings, highest_gains = recipient_view_context.off_chain_actively_sent_received_amounts(
eon_number=current_swap.eon_number, only_appended=False)
finalization_active_state = ActiveState(
wallet=current_swap.recipient,
updated_spendings=highest_spendings +
current_swap.amount_swapped,
updated_gains=highest_gains + current_swap.amount_swapped,
tx_set_hash=tx_set_hash,
tx_set_proof_hashes=transfer_proof,
tx_set_index=transfer_index,
eon_number=current_swap.eon_number)
finalization_active_state_signature_data = finalization_signatures[
0]
finalization_active_state_checksum = crypto.hex_value(
finalization_active_state.checksum())
finalization_active_state_signature = Signature(
wallet=current_swap.recipient,
checksum=finalization_active_state_checksum,
value=finalization_active_state_signature_data.get('value'))
if not finalization_active_state_signature.is_valid():
raise serializers.ValidationError(
detail='', code=ErrorCode.INVALID_CREDIT_SIGNATURE)
finalization_active_state_signature_records.append(
finalization_active_state_signature)
finalization_active_state_records.append(finalization_active_state)
# calculate future spent, gained, empty tx set
future_spent_gained = max(highest_spendings,
highest_gains) + swap.amount_swapped + 1
empty_tx_set_hash = crypto.hex_value(NODE_CACHE[0]['hash'])
for index in range(1, len(swap_set)):
future_swap = swap_set[index]
finalization_active_state = ActiveState(
wallet=future_swap.recipient,
updated_spendings=future_spent_gained,
updated_gains=future_spent_gained,
tx_set_hash=empty_tx_set_hash,
# any dummy value
tx_set_proof_hashes='',
# any dummy value
tx_set_index=0,
eon_number=future_swap.eon_number)
finalization_active_state_checksum = crypto.hex_value(
finalization_active_state.checksum())
finalization_active_state_signature = Signature(
wallet=swap.recipient,
checksum=finalization_active_state_checksum,
value=finalization_signatures[index].get('value'))
if not finalization_active_state_signature.is_valid():
raise serializers.ValidationError(
detail='',
code=ErrorCode.INVALID_FUTURE_CREDIT_SIGNATURE)
finalization_active_state_signature_records.append(
finalization_active_state_signature)
finalization_active_state_records.append(
finalization_active_state)
Signature.objects.bulk_create(
finalization_active_state_signature_records)
with current_swap.lock(
auto_renewal=False), current_swap.wallet.lock(
auto_renewal=False), current_swap.recipient.lock(
auto_renewal=False):
for index in range(len(finalization_active_state_records)):
finalization_active_state_records[
index].wallet_signature = finalization_active_state_signature_records[
index]
ActiveState.objects.bulk_create(
finalization_active_state_records)
for index in range(len(swap_set)):
swap_set[
index].recipient_finalization_active_state = finalization_active_state_records[
index]
if index > 0:
swap_set[index].voided = True
swap_set[index].appended = False
swap_set[index].processed = True
Transfer.objects.bulk_update(swap_set, [
'recipient_finalization_active_state', 'voided',
'appended', 'processed'
])
swap_set[0].sign_swap_finalization(
settings.HUB_OWNER_ACCOUNT_ADDRESS,
settings.HUB_OWNER_ACCOUNT_KEY)
swap_set[0].close(complete=True, appended=True)
current_swap = swap_set[0]
is_swap_finalized = True
if is_swap_finalized:
operator_celery.send_task('auditor.tasks.on_swap_finalization',
args=[current_swap.id])
return current_swap
def create(self, validated_data):
wallet = validated_data.pop('wallet')
recipient = validated_data.pop('recipient')
if not TokenPair.objects.filter(token_from=wallet.token,
token_to=recipient.token).exists():
raise serializers.ValidationError(
detail='', code=ErrorCode.TOKEN_PAIR_BLOCKED)
# swap data
valid_eons = validated_data.pop('valid_eons')
swap_amount = validated_data.pop('amount')
swap_nonce = validated_data.pop('nonce')
sell_order = validated_data.pop('sell_order', True)
swap_amount_swapped = validated_data.pop('amount_swapped')
debit_signatures = validated_data.pop('debit_signature')
debit_balance_signatures = validated_data.pop(
'debit_balance_signature')
credit_balance_signatures = validated_data.pop(
'credit_balance_signature')
credit_signatures = validated_data.pop('credit_signature')
recipient_fulfillment_signatures = validated_data.pop(
'credit_fulfillment_signature')
# common transaction id
tx_id = uuid.uuid4()
tx_time = timezone.now()
# cached items to be used later
sender_available_balance = 0
recipient_available_balance = 0
swap_set = []
debit_tx_set_index = []
credit_tx_set_index = []
# recipient_fulfillment_tx_set_index = []
debit_tx_set_cache = []
credit_tx_set_cache = []
# recipient_fulfillment_tx_set_cache = []
debit_balance_signature_records = []
credit_balance_signature_records = []
debit_signature_records = []
credit_signature_records = []
recipient_fulfillment_signature_records = []
debit_balance_records = []
credit_balance_records = []
debit_active_state_records = []
credit_active_state_records = []
recipient_fulfillment_active_state_records = []
initial_swap_confirmed = False
# get current eon
current_eon = LocalViewInterface.latest().eon_number()
# initial swap eon should be the current eon number
if validated_data.pop('eon_number') != current_eon:
raise serializers.ValidationError(
detail='', code=ErrorCode.EON_NUMBER_OUT_OF_SYNC)
wallets = sorted([wallet, recipient], key=lambda w: w.token.id)
with RootCommitment.read_write_lock(
suffix=current_eon, auto_renewal=False), wallets[0].lock(
auto_renewal=False), wallets[1].lock(auto_renewal=False):
if RootCommitment.objects.filter(eon_number=current_eon +
1).exists():
raise serializers.ValidationError(
detail='Checkpoint was already created for this eon.',
code=ErrorCode.EON_NUMBER_OUT_OF_SYNC)
for eon_number in range(current_eon, current_eon + valid_eons):
swap = Transfer(tx_id=tx_id,
wallet=wallet,
amount=swap_amount,
eon_number=eon_number,
recipient=recipient,
nonce=swap_nonce,
amount_swapped=swap_amount_swapped,
swap=True,
sell_order=sell_order,
time=tx_time)
wallet_view_context = WalletTransferContext(wallet=wallet,
transfer=swap)
recipient_view_context = WalletTransferContext(
wallet=recipient, transfer=swap)
if eon_number == current_eon:
# Ensure sender log consistency
can_append_to_sender_log = wallet_view_context.can_schedule_transfer(
)
if can_append_to_sender_log is not True:
raise serializers.ValidationError(
detail='Sender: {}'.format(
can_append_to_sender_log),
code=ErrorCode.DEBIT_WALLET_CANNOT_ADD_TRANSACTION)
# Ensure recipient log consistency
can_append_to_recipient_log = recipient_view_context.can_schedule_transfer(
)
if can_append_to_recipient_log is not True:
raise serializers.ValidationError(
detail='Recipient: {}'.format(
can_append_to_recipient_log),
code=ErrorCode.CREDIT_WALLET_CANNOT_ADD_TRANSACTION
)
# Ensure swap consistency
can_spend, sender_available_balance = wallet_view_context.can_send_transfer(
current_eon_number=current_eon,
using_only_appended_funds=False)
if can_spend is not True:
raise serializers.ValidationError(
detail=can_spend,
code=ErrorCode.DEBIT_WALLET_OVERSPENDING)
# Ensure that sender balance is exactly equal to total outgoing amount
if sender_available_balance != swap.amount:
raise serializers.ValidationError(
detail=
'Sender balance should be exactly equal to outgoing swap amount, {} != {}.'
.format(sender_available_balance, swap.amount),
code=ErrorCode.DEBIT_WALLET_BALANCE_AMOUNT_MISMATCH
)
# Ensure that recipient balance is zero
recipient_available_balance = recipient_view_context.available_funds_at_eon(
eon_number=eon_number, only_appended=False)
if recipient_available_balance != 0:
raise serializers.ValidationError(
detail='Recipient balance should be exactly zero.',
code=ErrorCode.CREDIT_WALLET_BALANCE_NOT_ZERO)
current_eon_swap = swap
sender_highest_spendings, sender_highest_gains = wallet_view_context.off_chain_actively_sent_received_amounts(
eon_number=swap.eon_number, only_appended=False)
recipient_highest_spendings, recipient_highest_gains = recipient_view_context.off_chain_actively_sent_received_amounts(
eon_number=swap.eon_number, only_appended=False)
else:
sender_highest_spendings, sender_highest_gains = 0, 0
recipient_highest_spendings, recipient_highest_gains = 0, 0
# Minimum Debit Balance Marker
debit_balance_marker_signature_data = debit_balance_signatures[
eon_number - current_eon]
debit_balance_marker = MinimumAvailableBalanceMarker(
wallet=wallet, eon_number=eon_number, amount=0)
debit_balance_marker_checksum = crypto.hex_value(
debit_balance_marker.checksum())
debit_balance_marker_signature = Signature(
wallet=wallet,
checksum=debit_balance_marker_checksum,
value=debit_balance_marker_signature_data.get('value'))
if not debit_balance_marker_signature.is_valid():
raise serializers.ValidationError(
detail='',
code=ErrorCode.INVALID_DEBIT_BALANCE_SIGNATURE)
# Minimum Credit Balance Marker
credit_balance_marker_signature_data = credit_balance_signatures[
eon_number - current_eon]
credit_balance_marker = MinimumAvailableBalanceMarker(
wallet=recipient, eon_number=eon_number, amount=0)
credit_balance_marker_checksum = crypto.hex_value(
credit_balance_marker.checksum())
credit_balance_marker_signature = Signature(
wallet=recipient,
checksum=credit_balance_marker_checksum,
value=credit_balance_marker_signature_data.get('value'))
if not credit_balance_marker_signature.is_valid():
raise serializers.ValidationError(
detail='',
code=ErrorCode.INVALID_CREDIT_BALANCE_SIGNATURE)
assert (sender_available_balance == swap.amount)
# Debit Authorization
debit_active_state_signature_data = debit_signatures[
eon_number - current_eon]
debit_active_state, \
debit_active_state_signature, \
debit_transfer_index, \
debit_transfer_cache = check_active_state_signature(
swap,
wallet,
debit_active_state_signature_data,
eon_number > current_eon,
sender_available_balance,
sender_highest_spendings,
sender_highest_gains,
signature_type=SignatureType.DEBIT)
# Credit Authorization
credit_active_state_signature_data = credit_signatures[
eon_number - current_eon]
credit_active_state, \
credit_active_state_signature, \
credit_transfer_index, \
credit_transfer_cache = check_active_state_signature(
swap,
recipient,
credit_active_state_signature_data,
eon_number > current_eon,
recipient_available_balance,
recipient_highest_spendings,
recipient_highest_gains,
signature_type=SignatureType.CREDIT)
# Finality Authorization
recipient_fulfillment_active_state_signature_data = recipient_fulfillment_signatures[
eon_number - current_eon]
recipient_fulfillment_active_state, \
recipient_fulfillment_active_state_signature, \
recipient_fulfillment_transfer_index, \
recipient_fulfillment_transfer_cache = check_active_state_signature(
swap,
recipient,
recipient_fulfillment_active_state_signature_data,
eon_number > current_eon,
recipient_available_balance,
recipient_highest_spendings,
recipient_highest_gains,
signature_type=SignatureType.FULFILLMENT)
# accumulate records to be saved
debit_balance_signature_records.append(
debit_balance_marker_signature)
credit_balance_signature_records.append(
credit_balance_marker_signature)
debit_signature_records.append(debit_active_state_signature)
credit_signature_records.append(credit_active_state_signature)
recipient_fulfillment_signature_records.append(
recipient_fulfillment_active_state_signature)
debit_balance_records.append(debit_balance_marker)
credit_balance_records.append(credit_balance_marker)
debit_active_state_records.append(debit_active_state)
credit_active_state_records.append(credit_active_state)
recipient_fulfillment_active_state_records.append(
recipient_fulfillment_active_state)
debit_tx_set_index.append(debit_transfer_index)
credit_tx_set_index.append(credit_transfer_index)
# recipient_fulfillment_tx_set_index.append(recipient_fulfillment_transfer_index)
debit_tx_set_cache.append(debit_transfer_cache)
credit_tx_set_cache.append(credit_transfer_cache)
# recipient_fulfillment_tx_set_cache.append(recipient_fulfillment_transfer_cache)
swap_set.append(swap)
assert (swap_set[0] is not None
and swap_set[0].eon_number == current_eon)
assert (len(swap_set) == valid_eons)
# locking context covers saving the state as well to make sure checkpoint creation is consistent
with transaction.atomic():
Signature.objects.bulk_create(
debit_balance_signature_records +
credit_balance_signature_records +
debit_signature_records + credit_signature_records +
recipient_fulfillment_signature_records)
for index in range(valid_eons):
debit_balance_records[
index].signature = debit_balance_signature_records[
index]
credit_balance_records[
index].signature = credit_balance_signature_records[
index]
debit_active_state_records[
index].wallet_signature = debit_signature_records[
index]
credit_active_state_records[
index].wallet_signature = credit_signature_records[
index]
recipient_fulfillment_active_state_records[
index].wallet_signature = recipient_fulfillment_signature_records[
index]
ActiveState.objects.bulk_create(
debit_active_state_records + credit_active_state_records +
recipient_fulfillment_active_state_records)
MinimumAvailableBalanceMarker.objects.bulk_create(
debit_balance_records + credit_balance_records)
for index in range(valid_eons):
swap_set[
index].sender_active_state = debit_active_state_records[
index]
swap_set[
index].recipient_active_state = credit_active_state_records[
index]
swap_set[
index].recipient_fulfillment_active_state = recipient_fulfillment_active_state_records[
index]
swap_set[
index].sender_balance_marker = debit_balance_records[
index]
swap_set[
index].sender_starting_balance = sender_available_balance
swap_set[
index].recipient_starting_balance = recipient_available_balance
# cache swap index in sender active set
swap_set[index].sender_merkle_index = debit_tx_set_index[
index]
# cache swap index in recipient active set
swap_set[
index].recipient_merkle_index = credit_tx_set_index[
index]
# cache active set merkle mountains height array and hash array for sender active set
swap_set[index].sender_merkle_hash_cache, swap_set[
index].sender_merkle_height_cache = debit_tx_set_cache[
index]
# cache active set merkle mountains height array and hash array for recipient active set
swap_set[index].recipient_merkle_hash_cache, swap_set[
index].recipient_merkle_height_cache = debit_tx_set_cache[
index]
Transfer.objects.bulk_create(swap_set)
swap_set[0].sign_swap(settings.HUB_OWNER_ACCOUNT_ADDRESS,
settings.HUB_OWNER_ACCOUNT_KEY)
initial_swap_confirmed = True
if initial_swap_confirmed:
operator_celery.send_task('auditor.tasks.on_swap_confirmation',
args=[swap_set[0].id])
return swap_set[0]
def create_token_commitment_for_eon(token: Token, eon_number):
logger.info('Creating Token Commitment for {} at {}'.format(
token.address, eon_number))
last_eon_number = eon_number - 1
with transaction.atomic():
wallets = Wallet.objects\
.filter(
token=token,
registration_operator_authorization__isnull=False,
trail_identifier__isnull=False)\
.order_by('trail_identifier')
new_balances = []
left, right = 0, 0
for wallet in wallets:
with wallet.lock(auto_renewal=True):
wallet_transfer_context = WalletTransferContext(
wallet=wallet, transfer=None)
last_transfer, last_transfer_is_outgoing = wallet_transfer_context.last_appended_active_transfer(
eon_number=last_eon_number)
last_transfer_active_state = None
if last_transfer is not None and last_transfer.is_open_swap():
last_transfer.retire_swap()
if last_transfer is not None:
last_transfer_active_state = WalletTransferContext.appropriate_transfer_active_state(
transfer=last_transfer,
is_outgoing=last_transfer_is_outgoing)
available_funds = wallet_transfer_context.available_funds_at_eon(
eon_number=last_eon_number,
only_appended=True)
right = left + available_funds
assert right >= left, 'Wallet {} Token {} Balance {}'.format(
wallet.address, token.address, available_funds)
passive_checksum, passive_amount, passive_marker = wallet_transfer_context.get_passive_values(
eon_number=last_eon_number)
new_balances.append({
'contract': settings.HUB_LQD_CONTRACT_ADDRESS,
'token': token.address,
'wallet': wallet.address,
'left': left,
'right': right,
'active_state_checksum': last_transfer_active_state.checksum() if last_transfer_active_state is not None else b'\0'*32,
'active_state': last_transfer_active_state,
'passive_checksum': passive_checksum,
'passive_amount': passive_amount,
'passive_marker': passive_marker,
})
left = right
last_incoming_passive_transfer = wallet_transfer_context.last_appended_incoming_passive_transfer(
eon_number=last_eon_number)
if last_incoming_passive_transfer:
wallet_transfer_context = WalletTransferContext(
wallet=wallet, transfer=last_incoming_passive_transfer)
passive_eon_transfers_list = wallet_transfer_context.incoming_passive_transfers_list(
only_appended=True,
force_append=False)
passive_transfers_merkle_tree = wallet_transfer_context.incoming_passive_transfers_tree(
only_appended=True,
force_append=False)
for index, incoming_passive_transfer in enumerate(passive_eon_transfers_list):
final_transfer_index = index
final_transfer_membership_proof = passive_transfers_merkle_tree.proof(
final_transfer_index)
final_transfer_membership_proof_chain = final_transfer_membership_proof.get(
"chain")
final_transfer_membership_proof_values = final_transfer_membership_proof.get(
"values")
assert incoming_passive_transfer.final_receipt_hashes is None
assert incoming_passive_transfer.final_receipt_index is None
assert incoming_passive_transfer.final_receipt_values is None
incoming_passive_transfer.final_receipt_hashes = final_transfer_membership_proof_chain
incoming_passive_transfer.final_receipt_index = final_transfer_index
incoming_passive_transfer.final_receipt_values = final_transfer_membership_proof_values
incoming_passive_transfer.save()
if last_transfer_active_state is None:
continue
wallet_transfer_context = WalletTransferContext(
wallet=wallet, transfer=last_transfer)
starting_balance = int(
wallet_transfer_context.starting_balance_in_eon(last_eon_number))
# if last active transfer is a multi eon swap
# starting balance included in every tx checksum should be set to the cached starting balance
# this way checkpoint state will match signed active state
if last_transfer.is_swap() and not last_transfer.cancelled:
if Transfer.objects.filter(eon_number=last_transfer.eon_number-1, tx_id=last_transfer.tx_id).exists():
matched_out, matched_in = last_transfer.matched_amounts(
all_eons=True)
current_matched_out, current_matched_in = last_transfer.matched_amounts(
all_eons=False)
if last_transfer_is_outgoing:
sender_starting_balance = last_transfer.sender_starting_balance
# current eon's starting balance should be equal to
# cached starting balance - committed matched out amount in past rounds
assert(starting_balance == sender_starting_balance -
matched_out + current_matched_out)
starting_balance = sender_starting_balance
else:
recipient_starting_balance = last_transfer.recipient_starting_balance
# current eon's starting balance should be equal to
# cached starting balance + committed matched in amount in past rounds
assert(
starting_balance == recipient_starting_balance + matched_in - current_matched_in)
starting_balance = recipient_starting_balance
confirmed_eon_transfers_list = wallet_transfer_context.authorized_transfers_list(
only_appended=True,
force_append=False)
confirmed_eon_transfers_list_shorthand = wallet_transfer_context.authorized_transfers_list_shorthand(
only_appended=True,
force_append=False,
last_transfer_is_finalized=False,
starting_balance=starting_balance)
transaction_merkle_tree = TransactionMerkleTree(
confirmed_eon_transfers_list_shorthand)
transaction_merkle_tree_root = hex_value(
transaction_merkle_tree.root_hash())
assert transaction_merkle_tree_root == last_transfer_active_state.tx_set_hash,\
'{}/{}'.format(transaction_merkle_tree_root,
last_transfer_active_state.tx_set_hash)
for confirmed_incoming_transfer in confirmed_eon_transfers_list:
if confirmed_incoming_transfer.recipient != wallet:
continue
final_transfer_index = transaction_merkle_tree.merkle_tree_nonce_map.get(
confirmed_incoming_transfer.nonce)
final_transfer_membership_proof_chain = transaction_merkle_tree.proof(
final_transfer_index)
assert confirmed_incoming_transfer.final_receipt_hashes is None
assert confirmed_incoming_transfer.final_receipt_index is None
confirmed_incoming_transfer.final_receipt_hashes = final_transfer_membership_proof_chain
confirmed_incoming_transfer.final_receipt_index = final_transfer_index
confirmed_incoming_transfer.save()
managed_funds = 0
if eon_number > 1:
last_eon = LocalViewInterface.confirmed(eon_number=last_eon_number)
pending_withdrawals_until_last_eon = \
WithdrawalRequest.objects\
.filter(wallet__token=token, eon_number__lte=last_eon_number, slashed=False)\
.filter(Q(withdrawal__isnull=True) | Q(withdrawal__block__gt=last_eon.block))
if not pending_withdrawals_until_last_eon.exists():
last_eon_pending_withdrawals = 0
else:
last_eon_pending_withdrawals = pending_withdrawals_until_last_eon\
.aggregate(Sum('amount')) \
.get('amount__sum')
total_token_balance = last_eon.contractledgerstate_set.get(
token=token).total_balance
managed_funds = total_token_balance - last_eon_pending_withdrawals
if right < managed_funds:
logger.warning('UNCLAIMED FUNDS: {} in {}'.format(
managed_funds - right, token.address))
send_admin_email(
subject='Soft TokenCommitment Warning: Extra funds',
content='There are some additional funds in the balance pool that belong to no one: {} of {}'
.format(managed_funds - right, token.address))
altered_balances = new_balances + [{
'contract': settings.HUB_LQD_CONTRACT_ADDRESS,
'token': token.address,
'wallet': settings.HUB_OWNER_ACCOUNT_ADDRESS,
'left': left,
'right': managed_funds,
'active_state_checksum': b'\0'*32,
'active_state': None,
'passive_checksum': b'\0'*32,
'passive_amount': 0,
'passive_marker': 0,
}]
new_merkle_tree = MerkleTree(altered_balances, managed_funds)
right = managed_funds
else:
if right > managed_funds:
logger.error('OVERCLAIMING FUNDS!! {} > {} in {}'.format(
right, managed_funds, token.address))
send_admin_email(
subject='HARD Checkpoint Error: OVERCLAIMING!',
content='OVERCLAIMING FUNDS!! {} > {} in {}'.format(right, managed_funds, token.address))
new_merkle_tree = MerkleTree(new_balances, right)
bulk_manager = BulkCreateManager(chunk_size=500)
for index, balance in enumerate(new_balances):
if not balance.get('wallet') or balance.get('wallet') == '0x0000000000000000000000000000000000000000':
continue
merkle_proof = new_merkle_tree.proof(index)
wallet = Wallet.objects.get(
token=token,
address=remove_0x_prefix(balance.get('wallet')))
# TODO verify validity through RPC prior to insertion
assert(wallet.trail_identifier == index)
# create records in batches
bulk_manager.add(
ExclusiveBalanceAllotment(
wallet=wallet,
eon_number=eon_number,
left=balance.get('left'),
right=balance.get('right'),
merkle_proof_hashes=merkle_proof.get('chain'),
merkle_proof_values=merkle_proof.get('values'),
merkle_proof_trail=index,
active_state=balance.get('active_state')
)
)
# make sure remaining batch is added
bulk_manager.done()
token_commitment = TokenCommitment.objects.create(
token=token,
merkle_root=hex_value(new_merkle_tree.root_hash()),
upper_bound=right)
return token_commitment
def create(self, validated_data):
active_state_signature_data = validated_data.pop('debit_signature')
wallet = validated_data.pop('wallet')
recipient = validated_data.pop('recipient')
# get current eon
current_eon = LocalViewInterface.latest().eon_number()
# transfer eon should be the current eon number
if validated_data.pop('eon_number') != current_eon:
raise serializers.ValidationError(
detail='', code=ErrorCode.EON_NUMBER_OUT_OF_SYNC)
# TODO refactor this such that the recipient is only locked after the sender's details are verified
wallets = sorted([wallet, recipient], key=lambda w: w.trail_identifier)
with RootCommitment.read_write_lock(
suffix=current_eon, auto_renewal=False), wallets[0].lock(
auto_renewal=False), wallets[1].lock(auto_renewal=False):
if RootCommitment.objects.filter(eon_number=current_eon +
1).exists():
raise serializers.ValidationError(
detail='', code=ErrorCode.EON_NUMBER_OUT_OF_SYNC)
transfer = Transfer(wallet=wallet,
amount=validated_data.pop('amount'),
eon_number=current_eon,
recipient=recipient,
nonce=validated_data.pop('nonce'),
passive=True)
wallet_view_context = WalletTransferContext(wallet=wallet,
transfer=transfer)
recipient_view_context = WalletTransferContext(wallet=recipient,
transfer=transfer)
# Minimal SLA
if not wallet.is_sla_exempt() and not recipient.is_sla_exempt():
if not wallet.has_valid_sla():
sender_transfers_list = wallet_view_context.authorized_transfers_list(
only_appended=False, force_append=True)
if len(sender_transfers_list) > settings.SLA_THRESHOLD:
raise serializers.ValidationError(
detail='',
code=ErrorCode.DEBIT_WALLET_EXCEEDED_SLA)
elif not recipient.has_valid_sla():
recipient_transfers_list = recipient_view_context.authorized_transfers_list(
only_appended=False, force_append=True)
if len(recipient_transfers_list) > settings.SLA_THRESHOLD:
raise serializers.ValidationError(
detail='',
code=ErrorCode.CREDIT_WALLET_EXCEEDED_SLA)
# Ensure sender log consistency
can_append_to_sender_log = wallet_view_context.can_schedule_transfer(
)
if can_append_to_sender_log is not True:
raise serializers.ValidationError(
detail='Sender: {}'.format(can_append_to_sender_log),
code=ErrorCode.DEBIT_WALLET_CANNOT_ADD_TRANSACTION)
# Ensure recipient log consistency
can_append_to_recipient_log = recipient_view_context.can_schedule_transfer(
)
if can_append_to_recipient_log is not True:
raise serializers.ValidationError(
detail='Recipient: {}'.format(can_append_to_recipient_log),
code=ErrorCode.CREDIT_WALLET_CANNOT_ADD_TRANSACTION)
# Ensure transfer consistency
can_spend, currently_available_funds = wallet_view_context.can_send_transfer(
current_eon_number=current_eon,
using_only_appended_funds=False)
if can_spend is not True:
raise serializers.ValidationError(
detail=can_spend, code=ErrorCode.DEBIT_WALLET_OVERSPENDING)
# Validate data
concise_balance_marker_signature_data = validated_data.pop(
'debit_balance_signature')
concise_balance_marker_amount = validated_data.pop('debit_balance')
if concise_balance_marker_amount > currently_available_funds - transfer.amount:
raise serializers.ValidationError(
detail='',
code=ErrorCode.DEBIT_WALLET_BALANCE_MARKER_EXCEED_BALANCE)
concise_balance_marker = MinimumAvailableBalanceMarker(
wallet=wallet,
eon_number=transfer.eon_number,
amount=concise_balance_marker_amount)
concise_balance_marker_checksum = hex_value(
concise_balance_marker.checksum())
concise_balance_marker_signature = Signature(
wallet=transfer.wallet,
checksum=concise_balance_marker_checksum,
value=concise_balance_marker_signature_data.get('value'))
if not concise_balance_marker_signature.is_valid():
raise serializers.ValidationError(
detail='', code=ErrorCode.INVALID_DEBIT_BALANCE_SIGNATURE)
tx_set_tree = wallet_view_context.optimized_authorized_transfers_tree(
)
tx_set_hash = hex_value(tx_set_tree.root_hash())
transfer_index = tx_set_tree.merkle_tree_nonce_map.get(
transfer.nonce)
transfer_proof = tx_set_tree.proof(transfer_index)
highest_spendings, highest_gains = wallet_view_context.off_chain_actively_sent_received_amounts(
eon_number=transfer.eon_number, only_appended=False)
active_state = ActiveState(wallet=wallet,
updated_spendings=highest_spendings +
transfer.amount,
updated_gains=highest_gains,
tx_set_hash=tx_set_hash,
tx_set_proof_hashes=transfer_proof,
tx_set_index=transfer_index,
eon_number=transfer.eon_number)
checksum = hex_value(active_state.checksum())
active_state_signature = Signature(
wallet=transfer.wallet,
checksum=checksum,
value=active_state_signature_data.get('value'))
if not active_state_signature.is_valid():
raise serializers.ValidationError(
detail='', code=ErrorCode.INVALID_DEBIT_SIGNATURE)
transfer.position = recipient_view_context.off_chain_passively_received_amount(
eon_number=transfer.eon_number, only_appended=False)
# locking context covers saving the state as well to make sure checkpoint creation is consistent
with transaction.atomic():
Signature.objects.bulk_create(
[concise_balance_marker_signature, active_state_signature])
concise_balance_marker.signature = concise_balance_marker_signature
concise_balance_marker.save()
active_state.wallet_signature = active_state_signature
active_state.operator_signature = active_state.sign_active_state(
settings.HUB_OWNER_ACCOUNT_ADDRESS,
settings.HUB_OWNER_ACCOUNT_KEY)
active_state.save()
transfer.sender_active_state = active_state
transfer.sender_balance_marker = concise_balance_marker
# cache transfer index in sender active set
transfer.sender_merkle_index = transfer_index
# transfer.sender_merkle_root_cache = tx_set_hash
# cache active set merkle mountains height array and hash array for recipient active set
transfer.sender_merkle_hash_cache, transfer.sender_merkle_height_cache = tx_set_tree.merkle_cache_stacks(
)
transfer.complete = True
transfer.appended = True
transfer.processed = True
transfer.save()
if transfer.appended:
operator_celery.send_task('auditor.tasks.on_transfer_confirmation',
args=[transfer.id])
return transfer
def concurrently_retrieve_state(contract_interface, contract_event_decoder,
verbose):
logger.info('Retrieve blocks.')
latest_chain_block = contract_interface.current_block() - 1
if not settings.DEBUG:
latest_chain_block += 1
logger.info('Latest chain block: {}'.format(latest_chain_block))
running_from = LocalViewInterface.latest_block() + 1
running_until = latest_chain_block
if running_from > running_until:
logger.info('No new blocks {}-{}.'.format(running_from, running_until))
return 0
confirm_from = LocalViewInterface.confirmed_block() + 1
confirm_until = running_until - contract_interface.get_blocks_for_confirmation(
)
if confirm_from > confirm_until:
logger.info('No new blocks to confirm.')
update_from = min(running_from, confirm_from)
update_until = min(update_from + 11, running_until + 1)
skipped = running_until + 1 - update_until
contract_state_tasks = []
contract_state_tasks_block_numbers = []
logger.info('Fetching [{},{})'.format(update_from, update_until))
for block_number in range(update_from, update_until):
if confirm_from <= block_number and block_number <= confirm_until:
contract_state_tasks.append(
fetch_confirmed_block.delay(block_number=block_number))
contract_state_tasks_block_numbers.append(block_number)
elif running_from <= block_number and block_number <= running_until:
contract_state_tasks.append(
fetch_running_block.delay(block_number=block_number))
contract_state_tasks_block_numbers.append(block_number)
for task_index, contract_state_task in enumerate(contract_state_tasks):
block_number = contract_state_tasks_block_numbers[task_index]
try:
task_result = contract_state_task.get(
timeout=settings.HUB_BLOCK_FETCH_TIMEOUT,
disable_sync_subtasks=False)
except exceptions.TimeoutError:
logger.error('Timed-out fetching block {}'.format(block_number))
for cleanup_index, task_to_clean_up in enumerate(
contract_state_tasks):
if cleanup_index >= task_index:
try:
task_to_clean_up.forget()
except NotImplementedError as e:
logger.error('Could not forget task results.')
logger.error(e)
break
if confirm_from <= block_number and block_number <= confirm_until:
confirmed_contract_state_dictionary, confirmed_contract_ledger_state_dictionaries, block_logs = task_result
confirmed_contract_state = ContractState.from_dictionary_form(
confirmed_contract_state_dictionary)
confirmed_ledger_states = [
ContractLedgerState.from_dictionary_form(
ledger_state, confirmed_contract_state) for ledger_state in
confirmed_contract_ledger_state_dictionaries
]
with transaction.atomic():
if running_from <= block_number and block_number <= running_until:
confirmed_contract_state.save()
for ledger_state in confirmed_ledger_states:
ledger_state.contract_state = confirmed_contract_state
ledger_state.save()
logger.info('Decoding logs for block {}.'.format(
confirmed_contract_state.block))
decoded_logs = contract_event_decoder.decode_many(block_logs)
eon_number = confirmed_contract_state.eon_number()
logger.info(
"Processing decoded logs in block %d eon %s: %d logs" %
(confirmed_contract_state.block, eon_number,
len(decoded_logs)))
for log in decoded_logs:
if log.get(u'name') in event_interpreter_map:
interpreter = event_interpreter_map.get(
log.get(u'name'))
interpreter.interpret(
decoded_event=log.get('data'),
txid=log.get('txid'),
block_number=confirmed_contract_state.block,
eon_number=eon_number,
verbose=verbose) if interpreter else None
else:
logger.error('UNKNOWN EVENT LOG {} '.format(log))
send_admin_email(
subject='Chain Sync Error: Unknown Log',
content='{}'.format(log))
running_contract_state = LocalViewInterface.running(
block_number=confirmed_contract_state.block)
if running_contract_state.confirm(confirmed_contract_state,
confirmed_ledger_states):
logger.info('Block {} confirmed.'.format(
confirmed_contract_state.block))
else:
logger.error('Block {} failed to confirm.'.format(
confirmed_contract_state.block))
send_admin_email(
subject='Chain Sync Confirmation Failure {}'.format(
confirmed_contract_state.block),
content='{}'.format(confirmed_contract_state))
raise Exception()
elif running_from <= block_number and block_number <= running_until:
logger.info('Process running block {}'.format(block_number))
confirmed_contract_state_dictionary, confirmed_contract_ledger_state_dictionaries = task_result
contract_state = ContractState.from_dictionary_form(
confirmed_contract_state_dictionary)
contract_state.save()
ledger_states = [
ContractLedgerState.from_dictionary_form(
ledger_state, contract_state) for ledger_state in
confirmed_contract_ledger_state_dictionaries
]
for ledger_state in ledger_states:
ledger_state.save()
logger.info('Running block {} stored.'.format(
contract_state.block))
else:
logger.info('Running from {} to {}.'.format(
running_from, running_until))
logger.info('Confirm from {} to {}.'.format(
confirm_from, confirm_until))
logger.info('Update from {} to {}.'.format(update_from,
update_until))
logger.error('Unexpected block number {}'.format(block_number))
send_admin_email(
subject='Chain Sync Unexpected Block {}'.format(block_number),
content='Out of order processing.')
raise Exception()
return skipped
def validate(self, attrs):
admission_requests = attrs.pop('admissions')
if len(admission_requests) > settings.BULK_ADMISSION_LIMIT:
raise serializers.ValidationError(
detail='Expected <= {} but got {} admisson requests.'.format(
settings.BULK_ADMISSION_LIMIT, len(admission_requests)),
code=ErrorCode.TOO_MANY_ADMISSION_REQUESTS)
registration_eon_number = LocalViewInterface.latest().eon_number()
latest_tos_config = TOSConfig.objects.all().order_by('time').last()
attrs['signatures'] = []
attrs['tos_signatures'] = []
attrs['wallets'] = []
all_tokens = {}
for token in Token.objects.all():
all_tokens[token.address.lower()] = token
for admission_request in admission_requests:
address = remove_0x_prefix(admission_request['address'])
token = all_tokens.get(
remove_0x_prefix(admission_request['token']).lower())
if token is None:
raise serializers.ValidationError(
detail='This token {} is not registered.'.format(
admission_request['token']),
code=ErrorCode.TOKEN_NOT_REGISTERED)
if BlacklistEntry.objects.filter(address__iexact=address).exists():
continue
if Wallet.objects.filter(address__iexact=address,
token=token).exists():
continue
wallet = Wallet(token=token,
address=address,
registration_eon_number=registration_eon_number)
admission_hash = hex_value(
wallet.get_admission_hash(registration_eon_number))
signature = Signature(
wallet=wallet,
checksum=admission_hash,
value=admission_request.get('authorization').get('value'))
if not signature.is_valid():
raise serializers.ValidationError(
detail='Invalid authorization for address {} and token {}.'
.format(admission_request['address'],
admission_request['token']),
code=ErrorCode.INVALID_ADMISSION_SIGNATURE)
tos_signature = Signature(
wallet=wallet,
checksum=latest_tos_config.digest(),
value=admission_request.get('tos_signature').get('value'))
if not tos_signature.is_valid():
raise serializers.ValidationError(
detail=
'Invalid TOS (digest: {}) signature for address {} and token {}.'
.format(latest_tos_config.digest(),
admission_request['address'],
admission_request['token']),
code=ErrorCode.INVALID_TOS_SIGNATURE)
attrs['signatures'].append(signature)
attrs['tos_signatures'].append(tos_signature)
attrs['wallets'].append(wallet)
attrs['tos_config'] = latest_tos_config
return attrs
