def send_signed_block(*, block, ip_address, port, protocol, url_path):
"""
Sign block and send to recipient
"""
network_signing_key = get_environment_variable('NETWORK_SIGNING_KEY')
signing_key = SigningKey(network_signing_key, encoder=HexEncoder)
node_identifier = get_verify_key(signing_key=signing_key)
node_identifier = encode_verify_key(verify_key=node_identifier)
message = sort_and_encode(block)
signed_block = {
'block': block,
'node_identifier': node_identifier,
'signature': generate_signature(message=message,
signing_key=signing_key)
}
node_address = format_address(ip_address=ip_address,
port=port,
protocol=protocol)
url = f'{node_address}{url_path}'
try:
post(url=url, body=signed_block)
except Exception as e:
logger.exception(e)
def send_invalid_block_to_banks(*, confirmation_block):
"""
Send invalid block to banks
This function is called by the confirmation validators only
"""
block = confirmation_block['block']
block_identifier = confirmation_block['block_identifier']
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator_node_identifier = self_configuration.primary_validator.node_identifier
self_configuration.primary_validator = None
self_configuration.save()
invalid_block = generate_signed_request(data={
'block':
block,
'block_identifier':
block_identifier,
'primary_validator_node_identifier':
primary_validator_node_identifier
},
nid_signing_key=get_signing_key())
for bank in get_banks_with_active_confirmation_services():
address = format_address(ip_address=bank.ip_address,
port=bank.port,
protocol=bank.protocol)
url = f'{address}/invalid_blocks'
try:
post(url=url, body=invalid_block)
except Exception as e:
logger.exception(e)
def test_banks_post_201_confirmation_validator(
client, validator_connection_request_data, signing_key,
self_configuration, requests_mock):
primary_validator = PrimaryValidatorConfigurationSerializer(
self_configuration.primary_validator)
validator_connection_request_data[
'primary_validator'] = primary_validator.data
address = format_address(
ip_address=validator_connection_request_data['ip_address'],
port=validator_connection_request_data.get('port'),
protocol=validator_connection_request_data['protocol'])
requests_mock.get(f'{address}/config',
json=validator_connection_request_data)
response = client.post_json(reverse('connection_requests-list'),
generate_signed_request(
data=validator_connection_request_data,
nid_signing_key=signing_key,
),
expected=status.HTTP_201_CREATED)
assert response == {}
assert Validator.objects.get(
ip_address=validator_connection_request_data['ip_address'])
def send_confirmation_block_history(*, block_identifier, ip_address, port,
protocol):
"""
Send historical confirmation blocks (starting with the block_identifier) to the confirmation validator
"""
address = format_address(ip_address=ip_address,
port=port,
protocol=protocol)
url = f'{address}/confirmation_blocks'
valid_confirmation_block = get_valid_confirmation_block(
block_identifier=block_identifier)
while valid_confirmation_block:
try:
post(url=url, body=valid_confirmation_block)
except Exception as e:
capture_exception(e)
logger.exception(e)
block_identifier = get_message_hash(
message=valid_confirmation_block['message'])
valid_confirmation_block = get_valid_confirmation_block(
block_identifier=block_identifier)
def send_primary_validator_updated_notices():
"""
Send a notice to all validators that the banks primary validator has been updated
- 200 response > validator is syncing to new primary validator
- 400 response > validator is not syncing to new primary validator (can be deleted)
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator = self_configuration.primary_validator
confirmation_validators = Validator.objects.all().exclude(node_identifier=primary_validator.node_identifier)
data = {
'ip_address': primary_validator.ip_address,
'port': primary_validator.port,
'protocol': primary_validator.protocol
}
for confirmation_validator in confirmation_validators:
signed_request = generate_signed_request(
data=data,
nid_signing_key=get_signing_key()
)
node_address = format_address(
ip_address=confirmation_validator.ip_address,
port=confirmation_validator.port,
protocol=confirmation_validator.protocol,
)
url = f'{node_address}/primary_validator_updated'
try:
post(url=url, body=signed_request)
except Exception as e:
confirmation_validator.delete()
logger.exception(e)
def create(self, validated_data):
"""
Handle banks primary validator updated notice
A response of True indicates to the requesting bank that self (this validator) will remain on the same network
Delete banks switching to different networks
"""
bank = validated_data['node_identifier']
ip_address = validated_data['ip_address']
port = validated_data['port']
protocol = validated_data['protocol']
self_configuration = get_self_configuration(
exception_class=RuntimeError)
if self.primary_validator_synchronized(
ip_address=ip_address, self_configuration=self_configuration):
return True
if (self_configuration.node_type == CONFIRMATION_VALIDATOR
and bank == get_most_trusted_bank()):
address = format_address(ip_address=ip_address,
port=port,
protocol=protocol)
try:
config = fetch(url=f'{address}/config', headers={})
except Exception as e:
capture_exception(e)
logger.exception(e)
else:
sync_with_primary_validator.delay(config=config)
return True
bank.delete()
raise serializers.ValidationError('Networks out of sync')
def get_node_config(data):
"""
Attempt to connect to node
Return nodes config data after validation
"""
ip_address = data['ip_address']
protocol = data['protocol']
try:
address = format_address(
ip_address=ip_address,
port=data.get('port'),
protocol=protocol
)
config_address = f'{address}/config'
config_data = fetch(url=config_address, headers={})
if config_data['node_type'] == BANK:
config_serializer = BankConfigurationSerializer(data=config_data)
elif config_data['node_type'] == CONFIRMATION_VALIDATOR:
config_serializer = ValidatorConfigurationSerializer(data=config_data)
elif config_data['node_type'] == PRIMARY_VALIDATOR:
raise serializers.ValidationError('Unable to accept connection requests from primary validators')
else:
raise serializers.ValidationError('Invalid node_type')
except Exception as e:
logger.exception(e)
raise e
if config_serializer.is_valid():
return config_data
else:
logger.exception(config_serializer.errors)
raise serializers.ValidationError(config_serializer.errors)
def send_signed_block(*, block, ip_address, port, protocol, url_path):
"""
Sign block and send to recipient
"""
signing_key = get_signing_key()
node_identifier = get_verify_key(signing_key=signing_key)
node_identifier = encode_verify_key(verify_key=node_identifier)
message = sort_and_encode(block)
signed_block = {
'block': block,
'node_identifier': node_identifier,
'signature': generate_signature(message=message,
signing_key=signing_key)
}
node_address = format_address(ip_address=ip_address,
port=port,
protocol=protocol)
url = f'{node_address}{url_path}'
try:
post(url=url, body=signed_block)
except Exception as e:
request_new_primary_validator()
logger.exception(e)
def send_connection_request(*, node, self_configuration):
"""
Send connection request to node
"""
node_address = format_address(
ip_address=node.ip_address,
port=node.port,
protocol=node.protocol,
)
signed_request = generate_signed_request(data={
'ip_address':
self_configuration.ip_address,
'port':
self_configuration.port,
'protocol':
self_configuration.protocol
},
nid_signing_key=get_signing_key())
url = f'{node_address}/connection_requests'
try:
post(url=url, body=signed_request)
except Exception as e:
logger.exception(e)
raise e
def send_confirmation_block_history_request():
"""
Request missing blocks from the primary validator
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator = self_configuration.primary_validator
address = format_address(
ip_address=primary_validator.ip_address,
port=primary_validator.port,
protocol=primary_validator.protocol
)
url = f'{address}/confirmation_block_history'
signed_request = generate_signed_request(
data={
'block_identifier': cache.get(HEAD_BLOCK_HASH)
},
nid_signing_key=get_signing_key()
)
try:
post(url=url, body=signed_request)
except Exception as e:
capture_exception(e)
logger.exception(e)
def start_crawl():
"""
Start a network crawl
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
self_node_identifier = self_configuration.node_identifier
primary_validator = self_configuration.primary_validator
primary_validator_address = format_address(
ip_address=primary_validator.ip_address,
port=primary_validator.port,
protocol=primary_validator.protocol)
crawl_banks(primary_validator_address=primary_validator_address,
self_node_identifier=self_node_identifier)
crawl_validators(primary_validator_address=primary_validator_address)
send_connection_requests(node_class=Bank,
self_configuration=self_configuration)
send_connection_requests(node_class=Validator,
self_configuration=self_configuration)
cache.set(CRAWL_LAST_COMPLETED, str(timezone.now()), None)
cache.set(CRAWL_STATUS, CRAWL_STATUS_NOT_CRAWLING, None)
send_crawl_status_notification()
def connect_to_primary_validator(*, primary_validator):
"""
Connect to a validator
- used in the syncing process
"""
self_configuration = get_self_configuration(exception_class=RuntimeError)
primary_validator_address = format_address(
ip_address=primary_validator.ip_address,
port=primary_validator.port,
protocol=primary_validator.protocol,
)
if is_connected_to_primary_validator(
primary_validator_address=primary_validator_address,
self_configuration=self_configuration):
return
signed_request = generate_signed_request(data={
'ip_address':
self_configuration.ip_address,
'port':
self_configuration.port,
'protocol':
self_configuration.protocol
},
nid_signing_key=get_signing_key())
url = f'{primary_validator_address}/connection_requests'
try:
post(url=url, body=signed_request)
except Exception as e:
logger.exception(e)
raise e
def create_validators(*, known_nodes, results):
"""
For each unknown validator, attempt to:
- fetch config data
- create new Validator object
"""
for validator in get_unknown_nodes(known_nodes=known_nodes,
results=results):
try:
address = format_address(ip_address=validator.get('ip_address'),
port=validator.get('port'),
protocol=validator.get('protocol'))
config_address = f'{address}/config'
config_data = fetch(url=config_address, headers={})
serializer = ValidatorConfigurationSerializer(data=config_data)
if serializer.is_valid():
create_validator_from_config_data(config_data=config_data)
continue
logger.exception(serializer.errors)
except Exception as e:
capture_exception(e)
logger.exception(e)
def get_primary_validator_address(self):
"""
Return formatted address of primary validator
"""
return format_address(ip_address=self.required_input['ip_address'],
port=self.required_input['port'],
protocol=self.required_input['protocol'])
def get_root_account_file(self, value=None):
"""
Get root account file from user
"""
valid = False
while not valid:
if self.unattended:
root_account_file = value
else:
root_account_file = input(
'Enter root account file URL (required): ')
if not root_account_file:
self._error('root_account_file required')
continue
try:
url_validator = URLValidator(schemes=['http', 'https'])
url_validator(root_account_file)
except ValidationError:
self._error('Invalid URL')
continue
if Path(root_account_file).suffix != '.json':
self._error('JSON file required')
continue
try:
download_root_account_file(url=root_account_file)
except Exception as e:
capture_exception(e)
logger.exception(e)
self.stdout.write(
self.style.ERROR(f'Error downloading {root_account_file}'))
self.stdout.write(self.style.ERROR(e))
continue
file_hash = get_file_hash(settings.ROOT_ACCOUNT_FILE_PATH)
if not self.required_input['head_block_hash']:
self.required_input['head_block_hash'] = file_hash
self_address = format_address(
ip_address=self.required_input.get('ip_address'),
port=self.required_input.get('port'),
protocol=self.required_input.get('protocol'),
)
root_account_file = get_root_account_file_url(address=self_address)
self.required_input.update({
'root_account_file': root_account_file,
'root_account_file_hash': file_hash
})
valid = True
def get_account_balance_lock(*, account_number):
"""
Return the balance lock for the given account
"""
bank_address = format_address(ip_address='192.168.1.75',
port=8000,
protocol='http')
url = f'{bank_address}/account_balance_lock/{account_number}'
results = fetch(url=url, headers={})
return results['balance_lock']
def get_account_balance_lock(*, account_number, live_pv=False):
"""
Return the balance lock for the given account
"""
if live_pv:
pv_address = format_address(
ip_address='64.225.47.205',
port=None,
protocol='http'
)
else:
pv_address = format_address(
ip_address='192.168.1.75',
port=8000,
protocol='http'
)
url = f'{pv_address}/account_balance_lock/{account_number}'
results = fetch(url=url, headers={})
return results['balance_lock']
def send_request_to_node(signed_request, live_pv=False):
"""
Send connection request to node
"""
if live_pv:
node_address = format_address(ip_address='64.225.47.205',
port=None,
protocol='http')
else:
node_address = format_address(ip_address='192.168.1.75',
port=8000,
protocol='http')
url = f'{node_address}/connection_requests'
results = post(url=url, body=signed_request)
if isinstance(results, dict):
for k, v in results.items():
print(f'{k}: {v}')
print(results)
def get_root_account_file_url(*, address=None):
"""Return root account file URL"""
if not address:
self_configuration = get_self_configuration(
exception_class=RuntimeError)
address = format_address(
ip_address=self_configuration.ip_address,
port=self_configuration.port,
protocol=self_configuration.protocol,
)
return address + default_storage.url(settings.ROOT_ACCOUNT_FILE_PATH)
def sync_to_new_primary_validator(*, ip_address, port, protocol):
"""
Sync to new primary validator (as directed by most trusted bank)
"""
address = format_address(ip_address=ip_address,
port=port,
protocol=protocol)
populate_confirmation_block_queue(
address=address,
error_handler=logger.exception,
initial_block_identifier=cache.get(HEAD_BLOCK_HASH))
process_confirmation_block_queue()
def test_clean_start_200_validator_removed(client, settings, requests_mock):
settings.CELERY_TASK_ALWAYS_EAGER = True
validator = Validator.objects.first()
validator_address = format_address(
ip_address=validator.ip_address,
port=validator.port,
protocol=validator.protocol
)
requests_mock.get(
f'{validator_address}/config',
json=ValidatorSerializer(validator).data,
)
clean_request(client, CLEAN_COMMAND_START, HTTP_200_OK)
validator.refresh_from_db()
def send_signed_post_request(*, data, ip_address, port, protocol, url_path):
"""Sign data and send to recipient"""
signed_request = generate_signed_request(data=data,
nid_signing_key=get_signing_key())
node_address = format_address(ip_address=ip_address,
port=port,
protocol=protocol)
url = f'{node_address}{url_path}'
try:
post(url=url, body=signed_request)
except Exception as e:
capture_exception(e)
logger.exception(e)
def fetch_valid_confirmation_block(*, primary_validator, block_identifier):
"""
Return valid confirmation block
"""
address = format_address(ip_address=primary_validator.ip_address,
port=primary_validator.port,
protocol=primary_validator.protocol)
url = f'{address}/valid_confirmation_blocks/{block_identifier}'
try:
results = fetch(url=url, headers={})
return results
except Exception as e:
logger.exception(e)
def is_self_known_to_node(*, node, self_configuration):
"""Return boolean to indicate if self is known to node"""
node_address = format_address(
ip_address=node.ip_address,
port=node.port,
protocol=node.protocol,
)
url = f'{node_address}/banks/{self_configuration.node_identifier}'
try:
fetch(url=url, headers={})
return True
except Exception as e:
logger.debug(e)
return False
def test_clean_start_200_bank_removed(client, settings, requests_mock):
settings.CELERY_TASK_ALWAYS_EAGER = True
bank = Bank.objects.first()
bank_address = format_address(
ip_address=bank.ip_address,
port=bank.port,
protocol=bank.protocol
)
requests_mock.get(
f'{bank_address}/config',
json=BankSerializer(bank).data,
)
clean_request(client, CLEAN_COMMAND_START, HTTP_200_OK)
with pytest.raises(Bank.DoesNotExist):
bank.refresh_from_db()
def send_confirmation_block_to_banks(*, confirmation_block):
"""
Send confirmed block to banks with active confirmation services
This function is called by the confirmation validators only
- primary validators send their confirmation blocks to the confirmation validators
"""
for bank in get_banks_with_active_confirmation_services():
address = format_address(ip_address=bank.ip_address,
port=bank.port,
protocol=bank.protocol)
url = f'{address}/confirmation_blocks'
try:
post(url=url, body=confirmation_block)
except Exception as e:
logger.exception(e)
def send_request_to_pv(signed_request):
"""
Send request to PV
"""
node_address = format_address(ip_address='64.225.47.205',
port=None,
protocol='http')
url = f'{node_address}/bank_blocks'
results = post(url=url, body=signed_request)
if isinstance(results, dict):
for k, v in results.items():
print(f'{k}: {v}')
print(results)
write_json(os.path.join(BLOCKS_DIR, 'bank-blocks-response.json'), results)
def test_banks_post_400_remote_response_confirmation_validator_no_primary_validator(
client, validator_connection_request_data, signing_key,
self_configuration, requests_mock):
address = format_address(
ip_address=validator_connection_request_data['ip_address'],
port=validator_connection_request_data.get('port'),
protocol=validator_connection_request_data['protocol'])
requests_mock.get(f'{address}/config',
json=validator_connection_request_data)
response = client.post_json(reverse('connection_requests-list'),
generate_signed_request(
data=validator_connection_request_data,
nid_signing_key=signing_key,
),
expected=status.HTTP_400_BAD_REQUEST)
assert response == {'primary_validator': ['This field is required.']}
def test_banks_post_400_invalid_node_type(client, bank_connection_request_data,
signing_key, self_configuration,
requests_mock):
bank_connection_request_data['node_type'] = 'BLAH_BLAH'
address = format_address(
ip_address=bank_connection_request_data['ip_address'],
port=bank_connection_request_data.get('port'),
protocol=bank_connection_request_data['protocol'])
requests_mock.get(f'{address}/config', json=bank_connection_request_data)
response = client.post_json(reverse('connection_requests-list'),
generate_signed_request(
data=bank_connection_request_data,
nid_signing_key=signing_key,
),
expected=status.HTTP_400_BAD_REQUEST)
assert response == {'non_field_errors': ['Invalid node_type']}
def send_block_to_bank(block):
"""
Send block to bank
"""
next_balance_lock = get_message_hash(message=block['message'])
print(f'\nNext balance lock will be: {next_balance_lock}\n')
bank_address = format_address(ip_address='167.99.173.247',
port=None,
protocol='http')
url = f'{bank_address}/blocks'
results = post(url=url, body=block)
if isinstance(results, dict):
for k, v in results.items():
print(f'{k}: {v}')
write_json(os.path.join(BLOCKS_DIR, 'blocks-response.json'), results)
