def test_node_client_get_state_metadata(tempfile_path):
# Add some node data
node_storage = CrawlerNodeStorage(storage_filepath=tempfile_path)
state_1 = create_random_mock_state(seed=1)
state_2 = create_random_mock_state(seed=2)
state_3 = create_random_mock_state(seed=3)
state_list = [state_1, state_2, state_3]
for state in state_list:
state_dict = FleetSensor.abridged_state_details(state)
node_storage.store_state_metadata(state=state_dict)
node_db_client = CrawlerStorageClient(db_filepath=tempfile_path)
result = node_db_client.get_previous_states_metadata(limit=len(state_list))
state_list.sort(key=lambda x: x.updated.epoch,
reverse=True) # sorted by timestamp in descending order
assert len(result) == len(state_list)
# verify result
# "result" of form of a list of state_info dictionaries
for idx, value in enumerate(result):
expected_row = convert_state_to_display_values(state_list[idx])
for info_idx, column in enumerate(CrawlerNodeStorage.STATE_DB_SCHEMA):
assert value[
column[0]] == expected_row[info_idx], f"{column[0]} matches"
def test_node_client_get_node_metadata(tempfile_path):
# Add some node data
node_storage = CrawlerNodeStorage(storage_filepath=tempfile_path)
node_1 = create_random_mock_node()
node_2 = create_random_mock_node()
node_3 = create_random_mock_node()
node_4 = create_random_mock_node()
node_5 = create_random_mock_node()
node_list = [node_1, node_2, node_3, node_4, node_5]
for node in node_list:
node_storage.store_node_metadata(node=node)
node_db_client = CrawlerStorageClient(db_filepath=tempfile_path)
result = node_db_client.get_known_nodes_metadata()
node_list.sort(
key=lambda x: x.checksum_address) # result is sorted by staker address
assert len(result) == len(node_list)
# "result" of form {staker_address -> {column_name -> column_value}}
for idx, key in enumerate(result):
node_info = result[key]
expected_row = convert_node_to_db_row(node_list[idx])
for info_idx, column in enumerate(CrawlerNodeStorage.NODE_DB_SCHEMA):
assert node_info[
column[0]] == expected_row[info_idx], f"{column[0]} matches"
def test_storage_deletion(tempfile_path):
assert os.path.exists(tempfile_path)
node_storage = CrawlerNodeStorage(storage_filepath=tempfile_path)
del node_storage
assert not os.path.exists(tempfile_path) # db file deleted
def test_storage_db_clear_not_metadata():
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
# store some data
node = create_random_mock_node()
node_storage.store_node_metadata(node=node)
state = create_specific_mock_state()
node_storage.store_state_metadata(state=state)
teacher_checksum = '0x123456789'
node_storage.store_current_teacher(teacher_checksum)
verify_all_db_tables(node_storage.db_conn, expect_empty=False)
# only clear certificates data
node_storage.clear(metadata=False, certificates=True)
# db tables should not have been cleared
verify_all_db_tables(node_storage.db_conn, expect_empty=False)
def test_storage_db_clear_only_metadata_not_certificates(sqlite_connection):
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
# store some data
node = create_random_mock_node()
node_storage.store_node_metadata(node=node)
state = create_specific_mock_state()
node_storage.store_state_metadata(state=FleetSensor.abridged_state_details(state))
teacher_checksum = '0x123456789'
node_storage.store_current_teacher(teacher_checksum)
verify_all_db_tables(sqlite_connection, expect_empty=False)
# clear metadata tables
node_storage.clear(metadata=True, certificates=False)
# db tables should have been cleared
verify_all_db_tables(sqlite_connection, expect_empty=True)
def test_storage_store_state_metadata():
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
state = create_specific_mock_state()
# Store state data
node_storage.store_state_metadata(
state=FleetStateTracker.abridged_state_details(state))
result = node_storage.db_conn.execute(
f"SELECT * FROM {CrawlerNodeStorage.STATE_DB_NAME}").fetchall()
assert len(result) == 1
for row in result:
verify_mock_state_matches_row(state, row)
# update state
new_now = state.updated.add(minutes=5)
new_color = 'red'
new_color_hex = '4F3D21'
symbol = '%'
updated_state = create_specific_mock_state(updated=new_now,
color=new_color,
color_hex=new_color_hex,
symbol=symbol)
node_storage.store_state_metadata(
state=FleetStateTracker.abridged_state_details(updated_state))
# ensure same item gets updated
result = node_storage.db_conn.execute(
f"SELECT * FROM {CrawlerNodeStorage.STATE_DB_NAME}").fetchall()
assert len(result) == 1 # state data is updated not added
for row in result:
verify_mock_state_matches_row(updated_state, row)
def test_storage_store_node_metadata():
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
node = create_specific_mock_node()
# Store node data
node_storage.store_node_metadata(node=node)
result = node_storage.db_conn.execute(
f"SELECT * FROM {CrawlerNodeStorage.NODE_DB_NAME}").fetchall()
assert len(result) == 1
for row in result:
verify_mock_node_matches(node, row)
# update node timestamp value and store
new_now = node.timestamp.add(hours=1)
worker_address = '0xabcdef'
updated_node = create_specific_mock_node(timestamp=new_now,
worker_address=worker_address)
# ensure same item gets updated
node_storage.store_node_metadata(node=updated_node)
result = node_storage.db_conn.execute(
f"SELECT * FROM {CrawlerNodeStorage.NODE_DB_NAME}").fetchall()
assert len(result) == 1 # node data is updated not added
for row in result:
verify_mock_node_matches(updated_node, row)
def test_storage_db_clear():
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
verify_all_db_tables_exist(node_storage.db_conn)
# store some data
node = create_random_mock_node()
node_storage.store_node_metadata(node=node)
state = create_specific_mock_state()
node_storage.store_state_metadata(
state=FleetStateTracker.abridged_state_details(state))
teacher_checksum = '0x123456789'
node_storage.store_current_teacher(teacher_checksum)
verify_all_db_tables(node_storage.db_conn, expect_empty=False)
# clear tables
node_storage.clear()
# db tables should have been cleared
verify_all_db_tables(node_storage.db_conn, expect_empty=True)
def store_node_db_data(storage: CrawlerNodeStorage, nodes: List, states: List):
for idx, node in enumerate(nodes):
storage.store_node_metadata(node=node)
if idx == 0:
# first item in list is teacher
storage.store_current_teacher(
teacher_checksum=node.checksum_address)
for state in states:
storage.store_state_metadata(state=state)
def test_storage_store_current_retrieval(sqlite_connection):
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
teacher_checksum = '0x123456789'
node_storage.store_current_teacher(teacher_checksum=teacher_checksum)
# check current teacher
verify_current_teacher(sqlite_connection, teacher_checksum)
# update current teacher
updated_teacher_checksum = '0x987654321'
node_storage.store_current_teacher(teacher_checksum=updated_teacher_checksum)
# check current teacher
verify_current_teacher(sqlite_connection, updated_teacher_checksum)
def test_node_client_get_current_teacher_checksum(tempfile_path):
node_storage = CrawlerNodeStorage(storage_filepath=tempfile_path)
teacher_checksum = '0x123456789'
node_storage.store_current_teacher(teacher_checksum=teacher_checksum)
node_db_client = CrawlerStorageClient(db_filepath=tempfile_path)
result = node_db_client.get_current_teacher_checksum()
assert result == teacher_checksum
new_teacher_checksum = '0x9876543221'
node_storage.store_current_teacher(teacher_checksum=new_teacher_checksum)
result = node_db_client.get_current_teacher_checksum()
assert result == new_teacher_checksum
def test_storage_store_state_metadata(sqlite_connection):
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
state = create_specific_mock_state()
# Store state data
node_storage.store_state_metadata(state=FleetSensor.abridged_state_details(state))
result = sqlite_connection.execute(f"SELECT * FROM {CrawlerNodeStorage.STATE_DB_NAME}").fetchall()
assert len(result) == 1
for row in result:
verify_mock_state_matches_row(state, row)
# update state
new_now = state.updated.add(minutes=5)
updated_state = create_specific_mock_state(updated=new_now)
node_storage.store_state_metadata(state=FleetSensor.abridged_state_details(updated_state))
# ensure same item gets updated
result = sqlite_connection.execute(f"SELECT * FROM {CrawlerNodeStorage.STATE_DB_NAME}").fetchall()
assert len(result) == 1 # state data is updated not added
for row in result:
verify_mock_state_matches_row(updated_state, row)
def test_storage_initialize():
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
node_storage.initialize() # re-initialize
verify_all_db_tables_exist(node_storage.db_conn)
def test_storage_init():
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
assert node_storage.db_filepath == IN_MEMORY_FILEPATH
assert not node_storage.federated_only
assert CrawlerNodeStorage._name != SQLiteForgetfulNodeStorage._name
def test_dashboard_render(new_blockchain_db_client, get_agent, tempfile_path,
dash_duo):
############## SETUP ################
current_period = 18622
# create node metadata
nodes_list, last_confirmed_period_dict = create_nodes(
num_nodes=5, current_period=current_period)
# create states
states_list = create_states(num_states=3)
# write node, teacher (first item in node list), and state data to storage
node_storage = CrawlerNodeStorage(storage_filepath=tempfile_path)
store_node_db_data(node_storage, nodes=nodes_list, states=states_list)
# Setup StakingEscrowAgent and ContractAgency
partitioned_stakers = (25, 5, 10) # confirmed, pending, inactive
global_locked_tokens = NU(1000000, 'NU').to_nunits()
staking_agent = create_mocked_staker_agent(
partitioned_stakers=partitioned_stakers,
current_period=current_period,
global_locked_tokens=global_locked_tokens,
last_confirmed_period_dict=last_confirmed_period_dict,
nodes_list=nodes_list)
contract_agency = MockContractAgency(staking_agent=staking_agent)
get_agent.side_effect = contract_agency.get_agent
# Setup Blockchain DB Client
historical_staked_tokens, historical_stakers, historical_work_orders = \
create_blockchain_db_historical_data(days_in_past=5)
mocked_blockchain_db_client = new_blockchain_db_client.return_value
configure_mocked_blockchain_db_client(
mocked_db_client=mocked_blockchain_db_client,
historical_tokens=historical_staked_tokens,
historical_stakers=historical_stakers,
historical_work_orders=historical_work_orders)
############## RUN ################
server = Flask("monitor-dashboard")
dashboard = monitor.dashboard.Dashboard(
flask_server=server,
route_url='/',
registry=None,
network='goerli',
influx_host='localhost',
influx_port=8086,
node_storage_db_filepath=tempfile_path)
dash_duo.start_server(dashboard.dash_app)
# check version
assert dash_duo.wait_for_element_by_id(
'version').text == f'v{nucypher.__version__}'
# check current period
assert dash_duo.wait_for_element_by_id('current-period-value').text == str(
current_period)
# check domain
assert dash_duo.wait_for_element_by_id('domain-value').text == 'goerli'
# check active ursulas
confirmed, pending, inactive = partitioned_stakers
assert dash_duo.wait_for_element_by_id('active-ursulas-value').text == \
f"{confirmed}/{confirmed + pending + inactive}"
# check staked tokens
assert dash_duo.wait_for_element_by_id('staked-tokens-value').text == str(
NU.from_nunits(global_locked_tokens))
#
# check dash components/tables - keeping it simple by simply checking text
# TODO there might be an easier way to test this
#
# staker breakdown
pie_chart_text = dash_duo.wait_for_element_by_id(
'staker-breakdown-graph').text
for num in partitioned_stakers:
assert str(num) in pie_chart_text
# historical and future graphs are difficult to test - values aren't available through selenium WebElement
# The revious tests (below) were incorrect - the `text` property only included the y-axis labels (not values)
# - just so happened the test had the values as the same
# -> Simply ensure that these graphs are loaded for now
historical_stakers = dash_duo.wait_for_element_by_id(
'prev-stakers-graph').text
historical_work_orders = dash_duo.wait_for_element_by_id(
'prev-orders-graph').text
future_locked_stake = dash_duo.wait_for_element_by_id('locked-graph').text
# check previous states
state_table = dash_duo.wait_for_element_by_id('state-table')
for state in states_list:
verify_state_data_in_table(state, state_table)
# check nodes
node_table = dash_duo.wait_for_element_by_id('node-table')
for node in nodes_list:
verify_node_data_in_table(
node=node,
last_confirmed_period=last_confirmed_period_dict[
node.checksum_address],
current_period=current_period,
node_table=node_table)
#
# test refresh/update buttons
#
# add a node and update page - ensure new node is displayed
new_node = create_random_mock_node(generate_certificate=False)
last_confirmed_period_dict[new_node.checksum_address] = current_period
nodes_list.append(new_node) # add new node to list
node_storage.store_node_metadata(new_node)
dash_duo.find_element("#node-update-button").click()
node_table_updated = dash_duo.wait_for_element_by_id('node-table')
# check for all nodes including new node
assert new_node in nodes_list, "ensure new node in list to check"
for node in nodes_list:
verify_node_data_in_table(
node=node,
last_confirmed_period=last_confirmed_period_dict[
node.checksum_address],
current_period=current_period,
node_table=node_table_updated)
# add a state and update page - ensure new state is displayed
new_state = create_random_mock_state()
states_list.append(new_state) # add state to list
node_storage.store_state_metadata(new_state)
dash_duo.find_element("#state-update-button").click()
state_table_updated = dash_duo.wait_for_element_by_id('state-table')
# check for all states including new state
assert new_state in states_list, "ensure new state in list to check"
for state in states_list:
verify_state_data_in_table(state, state_table_updated)
def test_storage_db_table_init(sqlite_connection):
node_storage = CrawlerNodeStorage(storage_filepath=IN_MEMORY_FILEPATH)
verify_all_db_tables_exist(sqlite_connection)
