def create(self, validated_data):
"""
If validator is more trusted than the current primary validator:
- switch to the new primary validator
If validator is less trusted than the current primary validator:
- delete the requesting validator (which is now on different network)
"""
validator = validated_data['node_identifier']
self_configuration = get_self_configuration(
exception_class=RuntimeError)
current_primary_validator = self_configuration.primary_validator
if current_primary_validator == validator:
return True
if current_primary_validator.trust < validator.trust:
self_configuration.primary_validator = validator
self_configuration.save()
send_primary_validator_updated_notices.delay()
send_primary_validator_updated_notification()
return True
validator.delete()
raise serializers.ValidationError('Networks out of sync')
def create(self, validated_data):
"""
Create block and bank transactions
Forward block to validator
"""
validated_block = validated_data
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator = self_configuration.primary_validator
try:
with transaction.atomic():
block = create_block_and_bank_transactions(validated_block)
Account.objects.get_or_create(
account_number=validated_block['account_number'],
defaults={'trust': 0},
)
send_signed_block.delay(
block=validated_block,
ip_address=primary_validator.ip_address,
port=primary_validator.port,
protocol=primary_validator.protocol,
url_path='/bank_blocks'
)
except Exception as e:
logger.exception(e)
raise serializers.ValidationError(e)
return block
def create(self, validated_data):
"""
Handle banks primary validator updated notice
A response of True indicates to the requesting bank that self (this validator) will remain on the same network
Delete banks switching to different networks
"""
bank = validated_data['node_identifier']
ip_address = validated_data['ip_address']
port = validated_data['port']
protocol = validated_data['protocol']
self_configuration = get_self_configuration(
exception_class=RuntimeError)
if self.primary_validator_synchronized(
ip_address=ip_address, self_configuration=self_configuration):
return True
if (self_configuration.node_type == CONFIRMATION_VALIDATOR
and bank == get_most_trusted_bank()):
address = format_address(ip_address=ip_address,
port=port,
protocol=protocol)
try:
config = fetch(url=f'{address}/config', headers={})
except Exception as e:
capture_exception(e)
logger.exception(e)
else:
sync_with_primary_validator.delay(config=config)
return True
bank.delete()
raise serializers.ValidationError('Networks out of sync')
def validate(self, data):
"""
Verify that correct payment exist for both Bank and Validator
"""
data = super(BlockSerializerCreate, self).validate(data)
account_number = data['account_number']
message = data['message']
txs = message['txs']
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator = get_primary_validator()
if account_number != self_configuration.account_number:
validate_transaction_exists(
amount=self_configuration.default_transaction_fee,
error=serializers.ValidationError,
recipient=self_configuration.account_number,
txs=txs
)
if account_number != primary_validator.account_number:
validate_transaction_exists(
amount=primary_validator.default_transaction_fee,
error=serializers.ValidationError,
recipient=primary_validator.account_number,
txs=txs
)
return data
def create(self, validated_data):
"""
Create block and bank transactions
Forward block to validator
"""
validated_block = validated_data
self_configuration = get_self_configuration(
exception_class=RuntimeError)
primary_validator = self_configuration.primary_validator
try:
with transaction.atomic():
block, created = create_block_and_related_objects(
validated_block)
send_signed_block.delay(
block=validated_block,
ip_address=primary_validator.ip_address,
port=primary_validator.port,
protocol=primary_validator.protocol,
url_path='/bank_blocks')
except serializers.ValidationError as e:
logger.exception(e)
raise e
except Exception as e:
logger.exception(e)
raise serializers.ValidationError(e)
return block
def send_primary_validator_updated_notices():
"""
Send a notice to all validators that the banks primary validator has been updated
- 200 response > validator is syncing to new primary validator
- 400 response > validator is not syncing to new primary validator (can be deleted)
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator = self_configuration.primary_validator
confirmation_validators = Validator.objects.all().exclude(node_identifier=primary_validator.node_identifier)
data = {
'ip_address': primary_validator.ip_address,
'port': primary_validator.port,
'protocol': primary_validator.protocol
}
for confirmation_validator in confirmation_validators:
signed_request = generate_signed_request(
data=data,
nid_signing_key=get_signing_key()
)
node_address = format_address(
ip_address=confirmation_validator.ip_address,
port=confirmation_validator.port,
protocol=confirmation_validator.protocol,
)
url = f'{node_address}/primary_validator_updated'
try:
post(url=url, body=signed_request)
except Exception as e:
confirmation_validator.delete()
logger.exception(e)
def inner(request, *args, **kwargs):
message = request.data.get('message')
node_identifier = request.data.get('node_identifier')
signature = request.data.get('signature')
self_configuration = get_self_configuration(
exception_class=RuntimeError)
if node_identifier != self_configuration.node_identifier:
return Response(status=status.HTTP_401_UNAUTHORIZED)
try:
verify_signature(message=sort_and_encode(message),
signature=signature,
verify_key=node_identifier)
except BadSignatureError as e:
logger.exception(e)
return Response({ERROR: BAD_SIGNATURE},
status=status.HTTP_401_UNAUTHORIZED)
except Exception as e:
logger.exception(e)
return Response({ERROR: UNKNOWN},
status=status.HTTP_401_UNAUTHORIZED)
return func(request, *args, **kwargs)
def validate_message(message):
"""
Check that Txs exist
Verify that correct payment exist for both Bank and Validator
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator = get_primary_validator()
bank_default_transaction_fee = self_configuration.default_transaction_fee
validator_transaction_fee = primary_validator.default_transaction_fee
txs = message['txs']
if not txs:
raise serializers.ValidationError('Invalid Txs')
validate_transaction_exists(
amount=bank_default_transaction_fee,
error=serializers.ValidationError,
recipient=self_configuration.account_number,
txs=txs
)
validate_transaction_exists(
amount=validator_transaction_fee,
error=serializers.ValidationError,
recipient=primary_validator.account_number,
txs=txs
)
return txs
def get_initial_block_identifier():
"""
Return initial block identifier (seed_block_identifier or root_account_file_hash)
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
return self_configuration.seed_block_identifier or self_configuration.root_account_file_hash
def start_crawl():
"""
Start a network crawl
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
self_node_identifier = self_configuration.node_identifier
primary_validator = self_configuration.primary_validator
primary_validator_address = format_address(
ip_address=primary_validator.ip_address,
port=primary_validator.port,
protocol=primary_validator.protocol)
crawl_banks(primary_validator_address=primary_validator_address,
self_node_identifier=self_node_identifier)
crawl_validators(primary_validator_address=primary_validator_address)
send_connection_requests(node_class=Bank,
self_configuration=self_configuration)
send_connection_requests(node_class=Validator,
self_configuration=self_configuration)
cache.set(CRAWL_LAST_COMPLETED, str(timezone.now()), None)
cache.set(CRAWL_STATUS, CRAWL_STATUS_NOT_CRAWLING, None)
send_crawl_status_notification()
def send_confirmation_block_history_request():
"""
Request missing blocks from the primary validator
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator = self_configuration.primary_validator
address = format_address(
ip_address=primary_validator.ip_address,
port=primary_validator.port,
protocol=primary_validator.protocol
)
url = f'{address}/confirmation_block_history'
signed_request = generate_signed_request(
data={
'block_identifier': cache.get(HEAD_BLOCK_HASH)
},
nid_signing_key=get_signing_key()
)
try:
post(url=url, body=signed_request)
except Exception as e:
capture_exception(e)
logger.exception(e)
def sync_from_primary_validators_initial_block(*, primary_validator):
"""
Sync from the primary validators initial block (seed_block_identifier or root_account_file_hash)
Invoked when self (confirmation validator):
- is first being initialized
- has a blockchain that is out of sync with the PV
"""
try:
self_configuration = get_self_configuration(exception_class=RuntimeError)
download_root_account_file(url=primary_validator.root_account_file)
self_configuration.root_account_file = get_root_account_file_url()
self_configuration.root_account_file_hash = get_file_hash(settings.ROOT_ACCOUNT_FILE_PATH)
self_configuration.seed_block_identifier = primary_validator.seed_block_identifier
self_configuration.save()
sync_accounts_table_to_root_account_file()
except Exception as e:
logger.exception(e)
raise RuntimeError(e)
rebuild_cache(head_block_hash=get_initial_block_identifier())
send_confirmation_block_history_request()
def sync_from_primary_validators_initial_block(*, primary_validator):
"""
Sync from the primary validators initial block (seed_block_identifier or root_account_file_hash)
Invoked when self (confirmation validator):
- is first being initialized
- has a blockchain that is out of sync with the PV
"""
try:
download_root_account_file(
url=primary_validator.root_account_file,
destination_file_path=settings.LOCAL_ROOT_ACCOUNT_FILE_PATH)
file_hash = get_file_hash(settings.LOCAL_ROOT_ACCOUNT_FILE_PATH)
# TODO: root_account_file should not be a copy of PVs URL but rather a unique path
# TODO: this way, every validator maintains their own copy
self_configuration = get_self_configuration(
exception_class=RuntimeError)
self_configuration.root_account_file = primary_validator.root_account_file
self_configuration.root_account_file_hash = file_hash
self_configuration.seed_block_identifier = primary_validator.seed_block_identifier
self_configuration.save()
sync_accounts_table_to_root_account_file()
except Exception as e:
logger.exception(e)
raise RuntimeError(e)
rebuild_cache(head_block_hash=get_initial_block_identifier())
send_confirmation_block_history_request()
def connect_to_primary_validator(*, primary_validator):
"""
Connect to a validator
- used in the syncing process
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator_address = format_address(
ip_address=primary_validator.ip_address,
port=primary_validator.port,
protocol=primary_validator.protocol,
)
if is_connected_to_primary_validator(
primary_validator_address=primary_validator_address,
self_configuration=self_configuration):
return
signed_request = generate_signed_request(data={
'ip_address':
self_configuration.ip_address,
'port':
self_configuration.port,
'protocol':
self_configuration.protocol
},
nid_signing_key=get_signing_key())
url = f'{primary_validator_address}/connection_requests'
try:
post(url=url, body=signed_request)
except Exception as e:
logger.exception(e)
raise e
def get_initial_block_identifier(self, primary_validator_config):
"""
Return initial block identifier
If seed_block_identifier, fetch related (seed) block and hash to get initial block identifier
Otherwise, return root_account_file_hash
"""
seed_block_identifier = primary_validator_config.get(
'seed_block_identifier')
if not seed_block_identifier:
self_configuration = get_self_configuration(
exception_class=RuntimeError)
root_account_file_hash = self_configuration.root_account_file_hash
pv_root_account_file_hash = primary_validator_config.get(
'root_account_file_hash')
if root_account_file_hash != pv_root_account_file_hash:
self._error(
'SelfConfiguration.root_account_file_hash does not match primary validator root_account_file_hash'
)
self._error(f'SelfConfiguration: {root_account_file_hash}')
self._error(f'Primary validator: {pv_root_account_file_hash}')
raise RuntimeError()
return root_account_file_hash
address = self.get_primary_validator_address()
confirmation_block = get_confirmation_block(
address=address, block_identifier=seed_block_identifier)
return get_message_hash(message=confirmation_block['message'])
def send_invalid_block_to_banks(*, confirmation_block):
"""
Send invalid block to banks
This function is called by the confirmation validators only
"""
block = confirmation_block['block']
block_identifier = confirmation_block['block_identifier']
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator_node_identifier = self_configuration.primary_validator.node_identifier
self_configuration.primary_validator = None
self_configuration.save()
invalid_block = generate_signed_request(data={
'block':
block,
'block_identifier':
block_identifier,
'primary_validator_node_identifier':
primary_validator_node_identifier
},
nid_signing_key=get_signing_key())
for bank in get_banks_with_active_confirmation_services():
address = format_address(ip_address=bank.ip_address,
port=bank.port,
protocol=bank.protocol)
url = f'{address}/invalid_blocks'
try:
post(url=url, body=invalid_block)
except Exception as e:
logger.exception(e)
def queued(request, pk):
self_configuration = get_self_configuration(exception_class=RuntimeError)
if self_configuration.node_type != CONFIRMATION_VALIDATOR:
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
return ConfirmationBlockViewSet.get_block(pk, get_queued_confirmation_block)
def handle(self, *args, **options):
"""
Run script
"""
valid_environments = ['local', 'postgres_local']
if ENVIRONMENT not in valid_environments:
raise RuntimeError(
f'DJANGO_APPLICATION_ENVIRONMENT must be in {valid_environments}'
)
ip = options['ip']
validate_ipv46_address(ip)
self.install_fixture_data()
self_configuration = get_self_configuration(
exception_class=RuntimeError)
SelfConfiguration.objects.filter(pk=self_configuration.id).update(
ip_address=ip)
rebuild_cache(
head_block_hash=self_configuration.root_account_file_hash)
self.stdout.write(
self.style.SUCCESS('Validator initialization complete'))
def validate(self, requesting_node_primary_validator_configuration):
"""
Validate that requesting nodes primary validator matches self primary validator
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
if self_configuration.node_type == PRIMARY_VALIDATOR:
self_primary_validator_configuration = SelfConfigurationSerializer(self_configuration).data
else:
primary_validator = self_configuration.primary_validator
self_primary_validator_configuration = ValidatorSerializer(primary_validator).data
for key in ['account_number', 'ip_address', 'node_identifier', 'protocol']:
requesting_node_value = requesting_node_primary_validator_configuration.get(key)
self_primary_validator_value = self_primary_validator_configuration.get(key)
if requesting_node_value is None:
raise serializers.ValidationError(f'{key} not found on requesting nodes primary validator')
if str(requesting_node_value) != str(self_primary_validator_value):
raise serializers.ValidationError(
f'Inconsistent primary validator settings for {key}. '
f'Requesting nodes value of {requesting_node_value} '
f'does not match expected value of {self_primary_validator_value}.'
)
return requesting_node_primary_validator_configuration
def create_confirmation_service(*, bank, confirmation_service_amount):
"""Create confirmation service for bank"""
current_confirmation_expiration = bank.confirmation_expiration
now = timezone.now()
if not current_confirmation_expiration:
start = now
else:
start = max([current_confirmation_expiration, now])
self_configuration = get_self_configuration(exception_class=RuntimeError)
daily_confirmation_rate = self_configuration.daily_confirmation_rate
confirmation_service_amount = int(confirmation_service_amount)
days_purchased = confirmation_service_amount / daily_confirmation_rate
seconds_purchased = days_purchased * 86400
seconds_purchased = int(seconds_purchased)
end = start + relativedelta(seconds=seconds_purchased)
BankConfirmationService.objects.create(bank=bank, end=end, start=start)
bank.confirmation_expiration = end
bank.save()
send_signed_post_request.delay(
data={
'end': str(end),
'start': str(start)
},
ip_address=bank.ip_address,
port=bank.port,
protocol=bank.protocol,
url_path='/validator_confirmation_services'
)
return seconds_purchased
def process_confirmation_block_queue():
"""
Process confirmation block queue
- this is for confirmation validators only
Ran after:
- initial sync with primary validator
- receiving confirmation block from the primary validator
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
head_block_hash = cache.get(HEAD_BLOCK_HASH)
confirmation_block = get_queued_confirmation_block(block_identifier=head_block_hash)
while confirmation_block:
block = confirmation_block['block']
is_valid, sender_account_balance = is_block_valid(block=block)
if not is_valid:
send_invalid_block_to_banks(confirmation_block=confirmation_block)
return
existing_accounts, new_accounts = get_updated_accounts(
sender_account_balance=sender_account_balance,
validated_block=block
)
if not updated_balances_match(
confirmation_block['updated_balances'],
format_updated_balances(existing_accounts, new_accounts)
):
send_invalid_block_to_banks(confirmation_block=confirmation_block)
return
update_accounts_cache(
existing_accounts=existing_accounts,
new_accounts=new_accounts
)
update_accounts_table(
existing_accounts=existing_accounts,
new_accounts=new_accounts
)
confirmation_block, head_block_hash = sign_block_to_confirm_and_update_head_block_hash(
block=block,
existing_accounts=existing_accounts,
new_accounts=new_accounts
)
delete_queued_confirmation_block(block_identifier=confirmation_block['message']['block_identifier'])
add_valid_confirmation_block(confirmation_block=confirmation_block)
if self_configuration.daily_confirmation_rate:
handle_bank_confirmation_services.delay(
block=block,
self_account_number=self_configuration.account_number
)
send_confirmation_block_to_banks(confirmation_block=confirmation_block)
confirmation_block = get_queued_confirmation_block(block_identifier=head_block_hash)
def confirmation_validator_configuration(monkeypatch):
load_validator_fixtures(CONFIRMATION_VALIDATOR_FIXTURES_DIR)
monkeypatch.setenv('NETWORK_SIGNING_KEY', '7a3359729b41f953d52818e787a312c8576e179e2ee50a2e4f28c4596b12dce0')
self_configuration = get_self_configuration(exception_class=RuntimeError)
rebuild_cache(head_block_hash=self_configuration.root_account_file_hash)
yield self_configuration
def set_primary_validator(*, validator):
"""Set validator as primary validator"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
self_configuration.primary_validator = validator
self_configuration.save()
connect_to_primary_validator(primary_validator=validator)
sync_blockchains(primary_validator=validator)
def validate(self, data):
"""Validate self is configured as primary validator"""
self_configuration = get_self_configuration(
exception_class=RuntimeError)
if self_configuration.node_type != PRIMARY_VALIDATOR:
raise serializers.ValidationError(
'Node is not configured as a primary validator')
return data
def request_new_primary_validator():
"""
Request a new primary validator
Called if/when the existing primary validator goes offline
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator = self_configuration.primary_validator
primary_validator.trust = 0
primary_validator.save()
set_primary_validator.delay()
def get_primary_validator():
"""Return primary validator"""
# TODO: This should be hitting the cache
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator = self_configuration.primary_validator
if not primary_validator:
raise RuntimeError('No primary validator')
return primary_validator
def set_primary_validator(*, validator):
"""Set validator as primary validator"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
self_configuration.primary_validator = validator
self_configuration.save()
if is_self_known_to_node(node=validator,
self_configuration=self_configuration):
return
send_connection_request(node=validator,
self_configuration=self_configuration)
def inner(obj, request, *args, **kwargs):
request, error = verify_request_signature(request=request, signed_data_key='message')
if error:
return Response(error, status=status.HTTP_401_UNAUTHORIZED)
node_identifier = request.data['node_identifier']
self_configuration = get_self_configuration(exception_class=RuntimeError)
if node_identifier != self_configuration.node_identifier:
return Response(status=status.HTTP_401_UNAUTHORIZED)
return func(obj, request, *args, **kwargs)
def create(request):
self_configuration = get_self_configuration(exception_class=RuntimeError)
if self_configuration.node_type != CONFIRMATION_VALIDATOR:
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
serializer = ConfirmationBlockSerializerCreate(data=request.data['message'])
serializer.is_valid(raise_exception=True)
serializer.save()
process_confirmation_block_queue.delay()
return Response({}, status=status.HTTP_201_CREATED)
def process_confirmation_block_queue():
"""
Process confirmation block queue
- this is for confirmation validators only
Ran after:
- initial sync with primary validator
- receiving confirmation block from the primary validator
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
queue = cache.get(CONFIRMATION_BLOCK_QUEUE)
head_block_hash = cache.get(HEAD_BLOCK_HASH)
confirmation_block = queue.pop(head_block_hash, None)
while confirmation_block:
block = confirmation_block['block']
is_valid, sender_account_balance = is_block_valid(block=block)
if not is_valid:
send_invalid_block_to_banks(confirmation_block=confirmation_block)
return
existing_accounts, new_accounts = get_updated_accounts(
sender_account_balance=sender_account_balance,
validated_block=block)
if not updated_balances_match(
confirmation_block['updated_balances'],
format_updated_balances(existing_accounts, new_accounts)):
send_invalid_block_to_banks(confirmation_block=confirmation_block)
return
update_accounts_cache(existing_accounts=existing_accounts,
new_accounts=new_accounts)
update_accounts_table(existing_accounts=existing_accounts,
new_accounts=new_accounts)
confirmation_block = sign_block_to_confirm(
block=block,
existing_accounts=existing_accounts,
new_accounts=new_accounts)
# TODO: Run as task
handle_bank_confirmation_services(
block=block, self_account_number=self_configuration.account_number)
send_confirmation_block_to_banks(confirmation_block=confirmation_block)
head_block_hash = cache.get(HEAD_BLOCK_HASH)
confirmation_block = queue.pop(head_block_hash, None)
cache.set(CONFIRMATION_BLOCK_QUEUE, queue, None)