def test_get_external_ip_from_known_nodes_with_one_known_node(
mock_requests, mock_network):
sensor = FleetSensor(domain=mock_network)
sensor._nodes['0xdeadbeef'] = Dummy()
assert len(sensor) == 1
get_external_ip_from_known_nodes(known_nodes=sensor)
# skipped because there are too few known nodes
mock_requests.assert_not_called()
def test_learner_restores_metadata_from_storage(lonely_ursula_maker, tmpdir):
# Create a local file-based node storage
root = tmpdir.mkdir("known_nodes")
metadata = root.mkdir("metadata")
certs = root.mkdir("certs")
old_storage = LocalFileBasedNodeStorage(federated_only=True,
metadata_dir=metadata,
certificates_dir=certs,
storage_root=root)
# Use the ursula maker with this storage so it's populated with nodes from one domain
_some_ursulas = lonely_ursula_maker(domain="fistro",
node_storage=old_storage,
know_each_other=True,
quantity=3,
save_metadata=True)
# Create a pair of new learners in a different domain, using the previous storage, and learn from it
new_learners = lonely_ursula_maker(domain="duodenal",
node_storage=old_storage,
quantity=2,
know_each_other=True,
save_metadata=False)
learner, buddy = new_learners
buddy._Learner__known_nodes = FleetSensor(domain="fistro")
# The learner shouldn't learn about any node from the first domain, since it's different.
learner.learn_from_teacher_node()
for restored_node in learner.known_nodes:
assert restored_node.mature().domain == learner.domain
# In fact, since the storage only contains nodes from a different domain,
# the learner should only know its buddy from the second domain.
assert set(learner.known_nodes) == {buddy}
def test_node_client_get_state_metadata(tempfile_path):
# Add some node data
node_storage = CrawlerNodeStorage(storage_filepath=tempfile_path)
state_1 = create_random_mock_state(seed=1)
state_2 = create_random_mock_state(seed=2)
state_3 = create_random_mock_state(seed=3)
state_list = [state_1, state_2, state_3]
for state in state_list:
state_dict = FleetSensor.abridged_state_details(state)
node_storage.store_state_metadata(state=state_dict)
node_db_client = CrawlerStorageClient(db_filepath=tempfile_path)
result = node_db_client.get_previous_states_metadata(limit=len(state_list))
state_list.sort(key=lambda x: x.updated.epoch,
reverse=True) # sorted by timestamp in descending order
assert len(result) == len(state_list)
# verify result
# "result" of form of a list of state_info dictionaries
for idx, value in enumerate(result):
expected_row = convert_state_to_display_values(state_list[idx])
for info_idx, column in enumerate(CrawlerNodeStorage.STATE_DB_SCHEMA):
assert value[
column[0]] == expected_row[info_idx], f"{column[0]} matches"
def test_blockchain_ursula_stamp_verification_tolerance(blockchain_ursulas, mocker):
#
# Setup
#
lonely_blockchain_learner, blockchain_teacher, unsigned, *the_others = list(blockchain_ursulas)
warnings = []
def warning_trapper(event):
if event['log_level'] == LogLevel.warn:
warnings.append(event)
# Make a bad identity evidence
unsigned._Teacher__decentralized_identity_evidence = unsigned._Teacher__decentralized_identity_evidence[:-5] + (b'\x00' * 5)
# Reset the metadata cache
unsigned._metadata = None
# Wipe known nodes!
lonely_blockchain_learner._Learner__known_nodes = FleetSensor(domain=TEMPORARY_DOMAIN)
lonely_blockchain_learner._current_teacher_node = blockchain_teacher
lonely_blockchain_learner.remember_node(blockchain_teacher)
globalLogPublisher.addObserver(warning_trapper)
lonely_blockchain_learner.learn_from_teacher_node(eager=True)
globalLogPublisher.removeObserver(warning_trapper)
# We received one warning during learning, and it was about this very matter.
assert len(warnings) == 1
warning = warnings[0]['log_format']
assert str(unsigned) in warning
assert "Verification Failed" in warning # TODO: Cleanup logging templates
# TODO: Buckets! #567
# assert unsigned not in lonely_blockchain_learner.known_nodes
# minus 2: self and the unsigned ursula.
# assert len(lonely_blockchain_learner.known_nodes) == len(blockchain_ursulas) - 2
assert blockchain_teacher in lonely_blockchain_learner.known_nodes
# Learn about a node with a badly signed payload
def bad_bytestring_of_known_nodes():
# Signing with the learner's signer instead of the teacher's signer
response_payload = MetadataResponsePayload(timestamp_epoch=blockchain_teacher.known_nodes.timestamp.epoch,
announce_nodes=[])
response = MetadataResponse(signer=lonely_blockchain_learner.stamp.as_umbral_signer(),
payload=response_payload)
return bytes(response)
mocker.patch.object(blockchain_teacher, 'bytestring_of_known_nodes', bad_bytestring_of_known_nodes)
globalLogPublisher.addObserver(warning_trapper)
lonely_blockchain_learner.learn_from_teacher_node(eager=True)
globalLogPublisher.removeObserver(warning_trapper)
assert len(warnings) == 2
warning = warnings[1]['log_format']
assert str(blockchain_teacher) in warning
assert "Failed to verify MetadataResponse from Teacher" in warning # TODO: Cleanup logging templates
def test_blockchain_alice_finds_ursula_via_rest(blockchain_alice, blockchain_ursulas):
# Imagine alice knows of nobody.
blockchain_alice._Learner__known_nodes = FleetSensor(domain=TEMPORARY_DOMAIN)
blockchain_alice.remember_node(blockchain_ursulas[0])
blockchain_alice.learn_from_teacher_node()
assert len(blockchain_alice.known_nodes) == len(blockchain_ursulas)
for ursula in blockchain_ursulas:
assert ursula in blockchain_alice.known_nodes
def test_get_external_ip_default_unknown_network():
unknown_domain = 'thisisnotarealdomain'
# Without fleet sensor
with pytest.raises(UnknownIPAddress):
determine_external_ip_address(network=unknown_domain)
# with fleet sensor
sensor = FleetSensor(domain=unknown_domain)
with pytest.raises(UnknownIPAddress):
determine_external_ip_address(known_nodes=sensor, network=unknown_domain)
def test_get_external_ip_cascade_failure(mocker, mock_network, mock_requests):
first = mocker.patch(
'nucypher.utilities.networking.get_external_ip_from_known_nodes',
return_value=None)
second = mocker.patch(
'nucypher.utilities.networking.get_external_ip_from_default_teacher',
return_value=None)
third = mocker.patch(
'nucypher.utilities.networking.get_external_ip_from_centralized_source',
return_value=None)
sensor = FleetSensor(domain=mock_network)
sensor._nodes['0xdeadbeef'] = Dummy()
with pytest.raises(UnknownIPAddress,
match='External IP address detection failed'):
determine_external_ip_address(network=mock_network, known_nodes=sensor)
first.assert_called_once()
second.assert_called_once()
third.assert_called_once()
def test_get_external_ip_from_known_nodes(mock_client, mock_network):
# Setup FleetSensor
sensor = FleetSensor(domain=mock_network)
sample_size = 3
sensor._nodes['0xdeadbeef'] = Dummy()
sensor._nodes['0xdeadllama'] = Dummy()
sensor._nodes['0xdeadmouse'] = Dummy()
assert len(sensor) == sample_size
# First sampled node replies
get_external_ip_from_known_nodes(known_nodes=sensor,
sample_size=sample_size)
assert mock_client.call_count == 1
mock_client.call_count = 0 # reset
# All sampled nodes dont respond
mock_client.return_value = Dummy.BadResponse
get_external_ip_from_known_nodes(known_nodes=sensor,
sample_size=sample_size)
assert mock_client.call_count == sample_size
def test_get_external_ip_from_known_nodes_client(mocker, mock_client,
mock_network):
# Setup FleetSensor
sensor = FleetSensor(domain=mock_network)
sample_size = 3
sensor._nodes['0xdeadbeef'] = Dummy()
sensor._nodes['0xdeadllama'] = Dummy()
sensor._nodes['0xdeadmouse'] = Dummy()
assert len(sensor) == sample_size
# Setup HTTP Client
mocker.patch.object(Ursula, 'from_teacher_uri', return_value=Dummy())
teacher_uri = RestMiddleware.TEACHER_NODES[mock_network][0]
get_external_ip_from_known_nodes(known_nodes=sensor,
sample_size=sample_size)
assert mock_client.call_count == 1 # first node responded
function, endpoint = mock_client.call_args[0]
assert function.__name__ == 'get'
assert endpoint == f'https://{teacher_uri}/ping'
def test_get_external_ip_from_known_nodes_with_one_known_node(mock_requests):
sensor = FleetSensor(domain=MOCK_NETWORK)
sensor.record_node(Dummy(b'deadbeefdeadbeefdead'))
sensor.record_fleet_state()
assert len(sensor) == 1
get_external_ip_from_known_nodes(known_nodes=sensor)
# skipped because there are too few known nodes
mock_requests.assert_not_called()
def test_blockchain_ursula_stamp_verification_tolerance(
blockchain_ursulas, mocker):
#
# Setup
#
lonely_blockchain_learner, blockchain_teacher, unsigned, *the_others = list(
blockchain_ursulas)
warnings = []
def warning_trapper(event):
if event['log_level'] == LogLevel.warn:
warnings.append(event)
#
# Attempt to verify unsigned stamp
#
unsigned._Teacher__decentralized_identity_evidence = NOT_SIGNED
# Wipe known nodes!
lonely_blockchain_learner._Learner__known_nodes = FleetSensor(
domain=TEMPORARY_DOMAIN)
lonely_blockchain_learner._current_teacher_node = blockchain_teacher
lonely_blockchain_learner.remember_node(blockchain_teacher)
globalLogPublisher.addObserver(warning_trapper)
lonely_blockchain_learner.learn_from_teacher_node(eager=True)
globalLogPublisher.removeObserver(warning_trapper)
# We received one warning during learning, and it was about this very matter.
assert len(warnings) == 1
warning = warnings[0]['log_format']
assert str(unsigned) in warning
assert "stamp is unsigned" in warning # TODO: Cleanup logging templates
# TODO: Buckets! #567
# assert unsigned not in lonely_blockchain_learner.known_nodes
# minus 2: self and the unsigned ursula.
# assert len(lonely_blockchain_learner.known_nodes) == len(blockchain_ursulas) - 2
assert blockchain_teacher in lonely_blockchain_learner.known_nodes
# Learn about a node with a badly signed payload
mocker.patch.object(lonely_blockchain_learner,
'verify_from',
side_effect=Learner.InvalidSignature)
lonely_blockchain_learner.learn_from_teacher_node(eager=True)
assert len(lonely_blockchain_learner.
suspicious_activities_witnessed['vladimirs']) == 1
def test_storage_store_state_metadata(sqlite_connection):
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
state = create_specific_mock_state()
# Store state data
node_storage.store_state_metadata(state=FleetSensor.abridged_state_details(state))
result = sqlite_connection.execute(f"SELECT * FROM {CrawlerNodeStorage.STATE_DB_NAME}").fetchall()
assert len(result) == 1
for row in result:
verify_mock_state_matches_row(state, row)
# update state
new_now = state.updated.add(minutes=5)
updated_state = create_specific_mock_state(updated=new_now)
node_storage.store_state_metadata(state=FleetSensor.abridged_state_details(updated_state))
# ensure same item gets updated
result = sqlite_connection.execute(f"SELECT * FROM {CrawlerNodeStorage.STATE_DB_NAME}").fetchall()
assert len(result) == 1 # state data is updated not added
for row in result:
verify_mock_state_matches_row(updated_state, row)
def test_get_external_ip_from_known_nodes(mock_client):
# Setup FleetSensor
sensor = FleetSensor(domain=MOCK_NETWORK)
sample_size = 3
sensor.record_node(Dummy('0xdeadbeef'))
sensor.record_node(Dummy('0xdeadllama'))
sensor.record_node(Dummy('0xdeadmouse'))
sensor.record_fleet_state()
assert len(sensor) == sample_size
# First sampled node replies
get_external_ip_from_known_nodes(known_nodes=sensor, sample_size=sample_size)
assert mock_client.call_count == 1
mock_client.call_count = 0 # reset
# All sampled nodes dont respond
mock_client.return_value = Dummy.BadResponse
get_external_ip_from_known_nodes(known_nodes=sensor, sample_size=sample_size)
assert mock_client.call_count == sample_size
def test_storage_db_clear_not_metadata(sqlite_connection):
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
# store some data
node = create_random_mock_node()
node_storage.store_node_metadata(node=node)
state = create_specific_mock_state()
node_storage.store_state_metadata(state=FleetSensor.abridged_state_details(state))
teacher_checksum = '0x123456789'
node_storage.store_current_teacher(teacher_checksum)
verify_all_db_tables(sqlite_connection, expect_empty=False)
# only clear certificates data
node_storage.clear(metadata=False, certificates=True)
# db tables should not have been cleared
verify_all_db_tables(sqlite_connection, expect_empty=False)
def test_get_external_ip_from_known_nodes_client(mocker, mock_client):
# Setup FleetSensor
sensor = FleetSensor(domain=MOCK_NETWORK)
sample_size = 3
sensor.record_node(Dummy(b'deadbeefdeadbeefdead'))
sensor.record_node(Dummy(b'deadllamadeadllamade'))
sensor.record_node(Dummy(b'deadmousedeadmousede'))
sensor.record_fleet_state()
assert len(sensor) == sample_size
# Setup HTTP Client
mocker.patch.object(Ursula,
'from_teacher_uri',
return_value=Dummy(b'deadporkdeadporkdead'))
teacher_uri = TEACHER_NODES[MOCK_NETWORK][0]
get_external_ip_from_known_nodes(known_nodes=sensor,
sample_size=sample_size)
assert mock_client.call_count == 1 # first node responded
function, endpoint = mock_client.call_args[0]
assert function.__name__ == 'get'
assert endpoint == f'https://{teacher_uri}/ping'
def test_get_external_ip_from_empty_known_nodes(mock_requests):
sensor = FleetSensor(domain=MOCK_NETWORK)
assert len(sensor) == 0
get_external_ip_from_known_nodes(known_nodes=sensor)
# skipped because there are no known nodes
mock_requests.assert_not_called()