def test_blockchain_client_close(new_influx_db):
mock_influxdb_client = new_influx_db.return_value
blockchain_db_client = CrawlerInfluxClient(None, None, None)
blockchain_db_client.close()
mock_influxdb_client.close.assert_called_once()
def test_blockchain_client_get_historical_locked_tokens(new_influx_db):
mock_influxdb_client = new_influx_db.return_value
mock_query_object = MagicMock(spec=ResultSet, autospec=True)
mock_influxdb_client.query.return_value = mock_query_object
# fake results for 5 days
days = 5
start_date = maya.now().subtract(days=days)
base_amount = 45000
amount_increment = 10000
results = []
for day in range(0, days):
results.append(dict(time=start_date.add(days=day).rfc3339(), sum=base_amount + (day * amount_increment)))
mock_query_object.get_points.return_value = results
blockchain_db_client = CrawlerInfluxClient(None, None, None)
locked_tokens_dict = blockchain_db_client.get_historical_locked_tokens_over_range(days)
# check query
today = datetime.utcnow()
range_end = datetime(year=today.year, month=today.month, day=today.day,
hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1) # include today in range
range_begin = range_end - timedelta(days=days)
expected_in_query = [
"SELECT SUM(locked_stake)",
"AS locked_stake",
f"FROM {Crawler.NODE_MEASUREMENT} WHERE time >= '{MayaDT.from_datetime(range_begin).rfc3339()}' AND "
f"time < '{MayaDT.from_datetime(range_end).rfc3339()}'",
"GROUP BY staker_address, time(1d)) GROUP BY time(1d)",
]
mock_influxdb_client.query.assert_called_once()
mock_query_object.get_points.assert_called_once()
call_args_list = mock_influxdb_client.query.call_args_list
assert len(call_args_list) == 1
for idx, execute_call in enumerate(call_args_list):
query = execute_call[0][0]
for statement in expected_in_query:
assert statement in query
# check results
assert len(locked_tokens_dict) == days
for idx, key in enumerate(locked_tokens_dict):
# use of rfc3339 loses milliseconds precision
date = MayaDT.from_rfc3339(start_date.add(days=idx).rfc3339()).datetime()
assert key == date
locked_tokens = locked_tokens_dict[key]
assert locked_tokens == base_amount + (idx * amount_increment)
# close must be explicitly called on CrawlerInfluxClient
mock_influxdb_client.close.assert_not_called()
def __init__(self, registry, flask_server: Flask, route_url: str,
network: str, crawler_host: str, crawler_port: int,
influx_host: str, influx_port: int):
self.log = Logger(self.__class__.__name__)
# Crawler
self.crawler_host = crawler_host
self.crawler_port = crawler_port
self.influx_client = CrawlerInfluxClient(
host=influx_host,
port=influx_port,
database=Crawler.INFLUX_DB_NAME)
# Blockchain & Contracts
self.network = network
self.registry = registry
# Agency
self.staking_agent = ContractAgency.get_agent(StakingEscrowAgent,
registry=self.registry)
self.token_agent = ContractAgency.get_agent(NucypherTokenAgent,
registry=self.registry)
self.policy_agent = ContractAgency.get_agent(PolicyManagerAgent,
registry=self.registry)
self.adjudicator_agent = ContractAgency.get_agent(
AdjudicatorAgent, registry=self.registry)
# Add informational endpoints
# Supply
self.add_supply_endpoint(flask_server=flask_server)
# TODO: Staker
# Dash
self.dash_app = self.make_dash_app(flask_server=flask_server,
route_url=route_url)
# GeoLocation
self.ip2loc = IP2Location.IP2Location()
self.ip2loc.open(
path.join(settings.ASSETS_PATH, 'geolocation',
'IP2LOCATION-LITE-DB5.BIN'))
class Dashboard:
"""
Dash Status application for monitoring a swarm of nucypher Ursula nodes.
"""
def __init__(self, registry, flask_server: Flask, route_url: str,
network: str, crawler_host: str, crawler_port: int,
influx_host: str, influx_port: int):
self.log = Logger(self.__class__.__name__)
# Crawler
self.crawler_host = crawler_host
self.crawler_port = crawler_port
self.influx_client = CrawlerInfluxClient(
host=influx_host,
port=influx_port,
database=Crawler.INFLUX_DB_NAME)
# Blockchain & Contracts
self.network = network
self.registry = registry
# Agency
self.staking_agent = ContractAgency.get_agent(StakingEscrowAgent,
registry=self.registry)
self.token_agent = ContractAgency.get_agent(NucypherTokenAgent,
registry=self.registry)
self.policy_agent = ContractAgency.get_agent(PolicyManagerAgent,
registry=self.registry)
self.adjudicator_agent = ContractAgency.get_agent(
AdjudicatorAgent, registry=self.registry)
# Dash
self.dash_app = self.make_dash_app(flask_server=flask_server,
route_url=route_url)
# GeoLocation
self.ip2loc = IP2Location.IP2Location()
self.ip2loc.open(
path.join(settings.ASSETS_PATH, 'geolocation',
'IP2LOCATION-LITE-DB5.BIN'))
def make_request(self):
url = f'http://{self.crawler_host}:{self.crawler_port}/{Crawler.METRICS_ENDPOINT}'
response = requests.get(url=url)
payload = response.json()
return payload
def verify_cached_stats(self, cached_stats):
if cached_stats is None:
# cached stats may not have been populated by the time it is attempted to be read from
# get data directly from the crawler - not expected to happen more than a few times during first page load
data = self.make_request()
else:
data = json.loads(cached_stats)
return data
def make_dash_app(self,
flask_server: Flask,
route_url: str,
debug: bool = False):
dash_app = Dash(
name=__name__,
server=flask_server,
assets_folder=settings.ASSETS_PATH,
url_base_pathname=route_url,
suppress_callback_exceptions=debug,
eager_loading=False,
assets_ignore='.*\\.BIN') # ignore ip2loc database file
# Initial State
dash_app.title = settings.TITLE
dash_app.layout = layout.BODY
@dash_app.callback(Output('header', 'children'),
[Input('url', 'pathname')]) # on page-load
def header(pathname):
return components.header()
@dash_app.callback(Output('cached-crawler-stats', 'children'),
[Input('request-interval', 'n_intervals')])
def update_cached_stats(n_intervals):
payload = self.make_request()
return json.dumps(payload)
@dash_app.callback(Output('prev-states', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def state(n_intervals, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
states = data['prev_states']
return components.previous_states(states=states)
@dash_app.callback(Output('network-info-content', 'children'), [
Input('url', 'pathname'),
Input('minute-interval', 'n_intervals'),
Input('network-info-tabs', 'value')
], [State('cached-crawler-stats', 'children')])
def network_info_tab_content(pathname, n, current_tab,
latest_crawler_stats):
if current_tab == 'node-details':
return known_nodes(latest_crawler_stats=latest_crawler_stats)
else:
return events()
def events():
prior_periods = 30 # TODO more thought? (note: retention for the db is 5w - so anything longer is useless)
events_data = self.influx_client.get_historical_events(
days=prior_periods)
events_table = components.events_table(events_data,
days=prior_periods)
return events_table
def known_nodes(latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
node_tables = components.known_nodes(
nodes_dict=data['node_details'])
return node_tables
@dash_app.callback(Output('active-stakers', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def active_stakers(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
data = data['activity']
confirmed, pending, inactive = data['active'], data[
'pending'], data['inactive']
total_stakers = confirmed + pending + inactive
return html.Div([
html.H4("Active Ursulas"),
html.H5(f"{confirmed}/{total_stakers}",
id='active-ursulas-value')
])
@dash_app.callback(Output('staker-breakdown', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def stakers_breakdown(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
return stakers_breakdown_pie_chart(data=data['activity'])
@dash_app.callback(Output('top-stakers-graph', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def top_stakers(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
return top_stakers_chart(data=data['top_stakers'])
@dash_app.callback(Output('current-period', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def current_period(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
return html.Div([
html.H4("Current Period"),
html.H5(data['current_period'], id='current-period-value')
])
@dash_app.callback(Output('blocktime-value', 'children'),
[Input('minute-interval', 'n_intervals')])
def blocktime(n):
# TODO: Consider doing this here or not - It exposes a web3 call on a public interface
block = self.staking_agent.blockchain.client.w3.eth.getBlock(
'latest')
blocktime = block.timestamp # epoch
blocktime = f"{MayaDT(blocktime).iso8601()} | {block.number}"
return html.Div(
[html.H4("Blocktime"),
html.H5(blocktime, id='blocktime')])
@dash_app.callback(Output('time-remaining', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def time_remaining(n, latest_crawler_stats):
# data = self.verify_cached_stats(latest_crawler_stats) # TODO: use period utils
tomorrow = datetime.now() + timedelta(1)
midnight = datetime(year=tomorrow.year,
month=tomorrow.month,
day=tomorrow.day,
hour=0,
minute=0,
second=0)
delta = (midnight - datetime.now())
slang = (maya.now() + delta).slang_time()
data = {'next_period': slang}
return html.Div(
[html.H4("Next Period"),
html.H5(data['next_period'])])
@dash_app.callback(Output('domains', 'children'),
[Input('url', 'pathname')]) # on page-load
def domains(pathname):
domains = f'{self.network.capitalize()} | {self.staking_agent.blockchain.client.chain_name}'
return html.Div(
[html.H4('Network'),
html.H5(domains, id="domain-value")])
@dash_app.callback(Output('registry', 'children'),
[Input('url', 'pathname')]) # on page-load
def registry(pathname):
latest = InMemoryContractRegistry.from_latest_publication(
network=self.network)
return html.Div([
html.H4('Registry'),
html.H5(latest.id[:16], id="registry-value")
])
@dash_app.callback(Output('contracts', 'children'),
[Input('url', 'pathname')]) # on page-load
def contracts(pathname):
agents = (self.token_agent, self.staking_agent, self.policy_agent,
self.adjudicator_agent)
rows = [make_contract_row(agent) for agent in agents]
_components = html.Div([html.H4('Contracts'), *rows],
id='contract-names')
return _components
@dash_app.callback(Output('staked-tokens', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def staked_tokens(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
staked = NU.from_nunits(data['global_locked_tokens'])
return html.Div([
html.H4('Staked Tokens'),
html.H5(f"{staked}", id='staked-tokens-value')
])
@dash_app.callback(Output('locked-stake-graph', 'children'),
[Input('daily-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def stake_and_known_nodes_plot(n, latest_crawler_stats):
prior_periods = 30
data = self.verify_cached_stats(latest_crawler_stats)
nodes_history = self.influx_client.get_historical_num_stakers_over_range(
prior_periods)
past_stakes = self.influx_client.get_historical_locked_tokens_over_range(
prior_periods)
future_stakes = data['future_locked_tokens']
graph = future_locked_tokens_bar_chart(
future_locked_tokens=future_stakes,
past_locked_tokens=past_stakes,
node_history=nodes_history)
return graph
@dash_app.callback(Output('nodes-geolocation-graph', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def nodes_geographical_locations(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
nodes_map = nodes_geolocation_map(nodes_dict=data['node_details'],
ip2loc=self.ip2loc)
return nodes_map
# @dash_app.callback(Output('prev-work-orders-graph', 'children'), [Input('daily-interval', 'n_intervals')])
# def historical_work_orders(n):
# TODO: only works for is_me characters
# prior_periods = 30
# num_work_orders_data = self.influx_client.get_historical_work_orders_over_range(prior_periods)
# return historical_work_orders_line_chart(data=num_work_orders_data)
return dash_app
class Dashboard:
"""
Dash Status application for monitoring a swarm of nucypher Ursula nodes.
"""
def __init__(self, registry, flask_server: Flask, route_url: str,
network: str, crawler_host: str, crawler_port: int,
influx_host: str, influx_port: int):
self.log = Logger(self.__class__.__name__)
# Crawler
self.crawler_host = crawler_host
self.crawler_port = crawler_port
self.influx_client = CrawlerInfluxClient(
host=influx_host,
port=influx_port,
database=Crawler.INFLUX_DB_NAME)
# Blockchain & Contracts
self.network = network
self.registry = registry
# Agency
self.staking_agent = ContractAgency.get_agent(StakingEscrowAgent,
registry=self.registry)
self.token_agent = ContractAgency.get_agent(NucypherTokenAgent,
registry=self.registry)
self.policy_agent = ContractAgency.get_agent(PolicyManagerAgent,
registry=self.registry)
self.adjudicator_agent = ContractAgency.get_agent(
AdjudicatorAgent, registry=self.registry)
# Add informational endpoints
# Supply
self.add_supply_endpoint(flask_server=flask_server)
# TODO: Staker
# Dash
self.dash_app = self.make_dash_app(flask_server=flask_server,
route_url=route_url)
# GeoLocation
self.ip2loc = IP2Location.IP2Location()
self.ip2loc.open(
path.join(settings.ASSETS_PATH, 'geolocation',
'IP2LOCATION-LITE-DB5.BIN'))
def make_request(self):
url = f'http://{self.crawler_host}:{self.crawler_port}/{Crawler.METRICS_ENDPOINT}'
response = requests.get(url=url)
payload = response.json()
return payload
def verify_cached_stats(self, cached_stats):
if cached_stats is None:
# cached stats may not have been populated by the time it is attempted to be read from
# get data directly from the crawler - not expected to happen more than a few times during first page load
data = self.make_request()
else:
data = json.loads(cached_stats)
return data
def add_supply_endpoint(self, flask_server: Flask):
@flask_server.route('/supply_information', methods=["GET"])
def supply_information():
economics = EconomicsFactory.retrieve_from_blockchain(
registry=self.registry)
parameter = request.args.get('q')
if parameter is None:
# no query - return all supply information
supply_info = calculate_supply_information(economics=economics)
response = flask_server.response_class(
response=json.dumps(supply_info),
status=200,
mimetype='application/json')
else:
# specific request query provided
if parameter == 'current_total_supply':
current_total_supply = calculate_current_total_supply(
economics)
response = flask_server.response_class(
response=str(current_total_supply),
status=200,
mimetype='text/plain')
elif parameter == 'est_circulating_supply':
est_circulating_supply = calculate_circulating_supply(
economics)
response = flask_server.response_class(
response=str(est_circulating_supply),
status=200,
mimetype='text/plain')
else:
response = flask_server.response_class(
response=f"Unsupported supply parameter: {parameter}",
status=400,
mimetype='text/plain')
return response
def make_dash_app(self,
flask_server: Flask,
route_url: str,
debug: bool = False):
dash_app = Dash(
name=__name__,
server=flask_server,
assets_folder=settings.ASSETS_PATH,
url_base_pathname=route_url,
suppress_callback_exceptions=debug,
eager_loading=False,
assets_ignore='.*\\.BIN') # ignore ip2loc database file
# Initial State
dash_app.title = settings.TITLE
dash_app.layout = layout.BODY
# TODO - not needed?
# @dash_app.callback(Output('header', 'children'), [Input('url', 'pathname')]) # on page-load
# def header(pathname):
# return components.header()
@dash_app.callback(Output('cached-crawler-stats', 'children'),
[Input('request-interval', 'n_intervals')])
def update_cached_stats(n_intervals):
payload = self.make_request()
return json.dumps(payload)
@dash_app.callback(Output('prev-states', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def state(n_intervals, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
states = data['prev_states']
return components.previous_states(states=states)
@dash_app.callback(Output('network-info-content', 'children'), [
Input('url', 'pathname'),
Input('minute-interval', 'n_intervals'),
Input('network-info-tabs', 'value')
], [State('cached-crawler-stats', 'children')])
def network_info_tab_content(pathname, n, current_tab,
latest_crawler_stats):
if current_tab == 'node-details':
return known_nodes(latest_crawler_stats=latest_crawler_stats)
else:
return events()
def events():
prior_periods = 30 # TODO more thought? (note: retention for the db is 5w - so anything longer is useless)
events_data = self.influx_client.get_historical_events(
days=prior_periods)
events_table = components.events_table(network=self.network,
events=events_data,
days=prior_periods)
return events_table
def known_nodes(latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
node_tables = components.known_nodes(
network=self.network, nodes_dict=data['node_details'])
return node_tables
@dash_app.callback(Output('active-stakers', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def active_stakers(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
data = data['activity']
confirmed, pending, inactive = data['active'], data[
'pending'], data['inactive']
total_stakers = confirmed + pending + inactive
return html.Div([
html.H4("Active Ursulas"),
html.H5(f"{confirmed}/{total_stakers}",
id='active-ursulas-value')
])
@dash_app.callback(Output('staker-breakdown', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def stakers_breakdown(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
return stakers_breakdown_pie_chart(data=data['activity'])
@dash_app.callback(Output('top-stakers-graph', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def top_stakers(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
return top_stakers_chart(data=data['top_stakers'])
@dash_app.callback(Output('current-period', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def current_period(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
return html.Div([
html.H4("Current Period"),
html.H5(data['current_period'], id='current-period-value')
])
@dash_app.callback(Output('blocktime-value', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def blocktime(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
block_epoch = data['blocktime']
block_number = data['blocknumber']
blocktime = f"{MayaDT(block_epoch).iso8601()} | {block_number}"
return html.Div(
[html.H4("Blocktime"),
html.H5(blocktime, id='blocktime')])
@dash_app.callback(Output('time-remaining', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def time_remaining(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
slang = MayaDT.from_iso8601(data['next_period']).slang_time()
return html.Div([html.H4("Next Period"), html.H5(slang)])
@dash_app.callback(Output('domain', 'children'),
[Input('url', 'pathname')]) # on page-load
def domain(pathname):
chain = self.staking_agent.blockchain.client.chain_name
network_and_chain = f'{self.network.capitalize()} | {chain}'
return html.Div([
html.H4('Network'),
html.H5(network_and_chain, id="domain-value")
])
@dash_app.callback(Output('registry', 'children'),
[Input('url', 'pathname')]) # on page-load
def registry(pathname):
latest = InMemoryContractRegistry.from_latest_publication(
network=self.network)
return html.Div([
html.H4('Registry'),
html.H5(latest.id[:16], id="registry-value")
])
@dash_app.callback(
Output('contracts', 'children'), [Input('domain', 'children')]
) # after domain obtained to prevent concurrent blockchain requests
def contracts(domain):
agents = (self.token_agent, self.staking_agent, self.policy_agent,
self.adjudicator_agent)
rows = [make_contract_row(self.network, agent) for agent in agents]
_components = html.Div([html.H4('Contracts'), *rows],
id='contract-names')
return _components
@dash_app.callback(Output('staked-tokens', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def staked_tokens(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
staked = NU.from_nunits(data['global_locked_tokens'])
return html.Div([
html.H4('Staked in Current Period'),
html.H5(f"{staked}", id='staked-tokens-value')
])
# @dash_app.callback(Output('locked-stake-graph', 'children'),
# [Input('daily-interval', 'n_intervals')],
# [State('cached-crawler-stats', 'children')])
# def stake_and_known_nodes_plot(n, latest_crawler_stats):
# prior_periods = 30
# data = self.verify_cached_stats(latest_crawler_stats)
# nodes_history = self.influx_client.get_historical_num_stakers_over_range(prior_periods)
# past_stakes = self.influx_client.get_historical_locked_tokens_over_range(prior_periods)
# future_stakes = data['future_locked_tokens']
# graph = future_locked_tokens_bar_chart(future_locked_tokens=future_stakes,
# past_locked_tokens=past_stakes,
# node_history=nodes_history)
# return graph
@dash_app.callback(Output('nodes-geolocation-graph', 'children'),
[Input('minute-interval', 'n_intervals')],
[State('cached-crawler-stats', 'children')])
def nodes_geographical_locations(n, latest_crawler_stats):
data = self.verify_cached_stats(latest_crawler_stats)
nodes_map = nodes_geolocation_map(nodes_dict=data['node_details'],
ip2loc=self.ip2loc)
return nodes_map
# @dash_app.callback(Output('prev-work-orders-graph', 'children'), [Input('daily-interval', 'n_intervals')])
# def historical_work_orders(n):
# TODO: only works for is_me characters
# prior_periods = 30
# num_work_orders_data = self.influx_client.get_historical_work_orders_over_range(prior_periods)
# return historical_work_orders_line_chart(data=num_work_orders_data)
return dash_app