def test_get_primary_no_epochs():
primary_oracle = PrimaryOracle()
primary_oracle.max_height = 4
for height in range(5):
for step in range(10):
with pytest.raises(ValueError):
primary_oracle.get_primary(height=height, step=step)
def primary_oracle(validators):
primary_oracle = PrimaryOracle()
primary_oracle.add_epoch(
Epoch(start_height=0,
validators=validators,
validator_definition_index=0))
primary_oracle.max_height = math.inf
return primary_oracle
def test_remove_irrelevant_epoch():
primary_oracle = PrimaryOracle()
primary_oracle.add_epoch(Epoch(0, [VALIDATOR1], 0))
primary_oracle.add_epoch(Epoch(5, [VALIDATOR2], 0))
primary_oracle.add_epoch(Epoch(2, [VALIDATOR3], 1))
primary_oracle.max_height = 5
assert primary_oracle.get_primary(height=5, step=0) == VALIDATOR3
def test_get_primary_late_epoch():
primary_oracle = PrimaryOracle()
primary_oracle.add_epoch(Epoch(5, [VALIDATOR1], 0))
primary_oracle.max_height = 4
for height in range(5):
for step in range(10):
with pytest.raises(ValueError):
primary_oracle.get_primary(height=height, step=step)
def _initialize_primary_oracle(self, chain_spec_path: Path) -> None:
with chain_spec_path.open("r") as f:
chain_spec = json.load(f)
self.original_chain_spec = chain_spec
validator_definition = chain_spec["engine"]["authorityRound"][
"params"]["validators"]
validator_definition_ranges = get_validator_definition_ranges(
validator_definition)
self.epoch_fetcher = EpochFetcher(self.w3,
validator_definition_ranges)
self.primary_oracle = PrimaryOracle()
static_epochs = get_static_epochs(validator_definition_ranges)
for epoch in static_epochs:
self.primary_oracle.add_epoch(epoch)
self._update_epochs()
def test_get_primary_single_epoch():
primary_oracle = PrimaryOracle()
primary_oracle.add_epoch(Epoch(0, [VALIDATOR1, VALIDATOR2], 0))
primary_oracle.max_height = 9
for height in range(10):
for step in range(0, 10, 2):
assert primary_oracle.get_primary(height=height,
step=step) == VALIDATOR1
for step in range(1, 10, 2):
assert primary_oracle.get_primary(height=height,
step=step) == VALIDATOR2
def test_epoch_sorting():
primary_oracle = PrimaryOracle()
primary_oracle.add_epoch(Epoch(0, [VALIDATOR1], 0))
primary_oracle.add_epoch(Epoch(10, [VALIDATOR3], 0))
primary_oracle.add_epoch(Epoch(5, [VALIDATOR2], 0))
primary_oracle.max_height = 14
for height in range(5):
assert primary_oracle.get_primary(height=height, step=0) == VALIDATOR1
for height in range(5, 10):
assert primary_oracle.get_primary(height=height, step=0) == VALIDATOR2
for height in range(10, 15):
assert primary_oracle.get_primary(height=height, step=0) == VALIDATOR3
def test_add_empty_epoch():
primary_oracle = PrimaryOracle()
with pytest.raises(ValueError):
primary_oracle.add_epoch(Epoch(0, [], 0))
def test_get_too_far_ahead():
primary_oracle = PrimaryOracle()
primary_oracle.add_epoch(Epoch(0, [VALIDATOR1], 0))
primary_oracle.max_height = 5
with pytest.raises(ValueError):
primary_oracle.get_primary(height=6, step=0)
class App:
logger = structlog.get_logger("monitor.main")
def __init__(
self,
*,
rpc_uri,
chain_spec_path,
report_dir,
db_path,
skip_rate,
offline_window_size,
initial_block_resolver,
upgrade_db=False,
watch_chain_spec=False,
):
self.report_dir = report_dir
self.skip_file = open(report_dir / SKIP_FILE_NAME, "a")
self.w3 = None
self.epoch_fetcher = None
self.primary_oracle = None
self.db = None
self.block_fetcher = None
self.skip_reporter = None
self.offline_reporter = None
self.equivocation_reporter = None
self.initial_block_resolver = initial_block_resolver
self.chain_spec_path = chain_spec_path
self.original_chain_spec = None
self.watch_chain_spec = watch_chain_spec
self._initialize_db(db_path)
self._initialize_w3(rpc_uri)
self.wait_for_node_fully_synced()
self._initialize_primary_oracle(chain_spec_path)
app_state = self._load_app_state()
if upgrade_db:
app_state = self._upgrade_app_state(app_state)
self._initialize_reporters(app_state, skip_rate, offline_window_size)
self._register_reporter_callbacks()
self._running = False
def wait_for_node_fully_synced(self):
def is_synced(node_status):
syncmsg = "still syncing" if node_status.is_syncing else "fully synced"
self.logger.info(f"node {syncmsg}:{node_status}")
return not node_status.is_syncing
node_status.wait_for_node_status(self.w3, is_synced)
def run(self) -> None:
self._running = True
try:
self.logger.info("starting sync")
while self._running:
self._run_cycle()
finally:
self.skip_file.close()
def _run_cycle(self) -> None:
self._update_epochs()
with self.db.persistent_session() as session:
number_of_new_blocks = self.block_fetcher.fetch_and_insert_new_blocks(
max_number_of_blocks=500,
max_block_height=self.epoch_fetcher.last_fetch_height,
)
self.db.store_pickled(APP_STATE_KEY, self.app_state)
self.skip_file.flush()
session.commit()
self.logger.info(
f"Syncing ({self.block_fetcher.get_sync_status():.0%})"
if self.block_fetcher.syncing else "Synced",
head=format_block(self.block_fetcher.head),
head_hash=self.block_fetcher.head.hash.hex(),
)
if number_of_new_blocks == 0:
time.sleep(BLOCK_FETCH_INTERVAL)
# check at the end of the cycle so that we quit immediately when the chain spec has
# changed
self._check_chain_spec()
def _check_chain_spec(self) -> None:
if not self.watch_chain_spec:
return
with self.chain_spec_path.open("r") as f:
try:
chain_spec = json.load(f)
except json.JSONDecodeError:
chain_spec_has_changed = True
else:
chain_spec_has_changed = chain_spec != self.original_chain_spec
if chain_spec_has_changed:
self.logger.info("Chain spec file has changed.")
self.stop()
def _update_epochs(self) -> None:
new_epochs = self.epoch_fetcher.fetch_new_epochs()
for epoch in new_epochs:
self.primary_oracle.add_epoch(epoch)
self.primary_oracle.max_height = self.epoch_fetcher.last_fetch_height
def stop(self):
self.logger.info(
"Stopping tlbc-monitor. This may take a long time, please be patient!"
)
self._running = False
@property
def app_state(self):
return AppStateV2(
block_fetcher_state=self.block_fetcher.state,
skip_reporter_state=self.skip_reporter.state,
offline_reporter_state=self.offline_reporter.state,
)
#
# Initialization
#
def _initialize_db(self, db_path):
db_url = SQLITE_URL_FORMAT.format(path=db_path)
engine = create_engine(db_url)
self.db = BlockDB(engine)
def _initialize_w3(self, rpc_uri):
self.w3 = Web3(HTTPProvider(rpc_uri))
# Inject custom middleware to improve the handling of connection
# problems with the RPC endpoint.
if "http_retry_request" in self.w3.middleware_onion:
self.w3.middleware_onion.replace(
"http_retry_request", http_retry_request_middleware_endlessly)
else:
self.w3.middleware_onion.add(
http_retry_request_middleware_endlessly)
def _initialize_primary_oracle(self, chain_spec_path: Path) -> None:
with chain_spec_path.open("r") as f:
chain_spec = json.load(f)
self.original_chain_spec = chain_spec
validator_definition = chain_spec["engine"]["authorityRound"][
"params"]["validators"]
validator_definition_ranges = get_validator_definition_ranges(
validator_definition)
self.epoch_fetcher = EpochFetcher(self.w3,
validator_definition_ranges)
self.primary_oracle = PrimaryOracle()
static_epochs = get_static_epochs(validator_definition_ranges)
for epoch in static_epochs:
self.primary_oracle.add_epoch(epoch)
self._update_epochs()
def _initialize_reporters(self, app_state, skip_rate, offline_window_size):
if not isinstance(app_state, AppStateV2):
raise InvalidAppStateException()
self.block_fetcher = BlockFetcher(
state=app_state.block_fetcher_state,
w3=self.w3,
db=self.db,
max_reorg_depth=MAX_REORG_DEPTH,
initial_block_resolver=self.initial_block_resolver,
)
self.skip_reporter = SkipReporter(
state=app_state.skip_reporter_state,
primary_oracle=self.primary_oracle,
grace_period=GRACE_PERIOD,
)
self.offline_reporter = OfflineReporter(
state=app_state.offline_reporter_state,
primary_oracle=self.primary_oracle,
offline_window_size=offline_window_size,
allowed_skip_rate=skip_rate,
)
self.equivocation_reporter = EquivocationReporter(db=self.db)
def _initialize_app_state(self):
self.logger.info("no state entry found, starting from fresh state")
return AppStateV2(
block_fetcher_state=BlockFetcher.get_fresh_state(),
skip_reporter_state=SkipReporter.get_fresh_state(),
offline_reporter_state=OfflineReporter.get_fresh_state(),
)
def _load_app_state(self):
"""Loads and returns the app state object. Make sure do initialize the db first"""
return self.db.load_pickled(
APP_STATE_KEY) or self._initialize_app_state()
def _upgrade_app_state(self, app_state):
if isinstance(app_state, AppStateV1):
self.logger.info("Upgrade appstate from v1 to v2")
return upgrade_v1_to_v2(app_state)
elif isinstance(app_state, AppStateV2):
return app_state
else:
raise InvalidAppStateException(
"Can not upgrade unsupported app state version")
def _register_reporter_callbacks(self):
self.block_fetcher.register_report_callback(self.skip_reporter)
self.block_fetcher.register_report_callback(self.equivocation_reporter)
self.skip_reporter.register_report_callback(self.skip_logger)
self.skip_reporter.register_report_callback(self.offline_reporter)
self.offline_reporter.register_report_callback(self.offline_logger)
self.equivocation_reporter.register_report_callback(
self.equivocation_logger)
#
# Reporters
#
def skip_logger(self, validator, skipped_proposal):
skip_timestamp = step_number_to_timestamp(skipped_proposal.step)
self.skip_file.write("{},{},{}\n".format(
skipped_proposal.step,
encode_hex(validator),
datetime.datetime.utcfromtimestamp(skip_timestamp),
))
def offline_logger(self, validator, steps):
filename = (
f"offline_report_{encode_hex(validator)}_steps_{min(steps)}_to_{max(steps)}"
)
with open(self.report_dir / filename, "w") as f:
json.dump(
{
"validator": encode_hex(validator),
"missed_steps": steps
}, f)
def equivocation_logger(self, equivocated_block_hashes):
"""Log a reported equivocation event.
Equivocation reports are logged into files separated by the proposers
address. Logged information are the proposer of the blocks, the steps
at which all blocks have been equivocated and a list of all block hashes
with their timestamp. Additionally two representing blocks are logged
with their RLP encoded header and related signature, which can be used
for an equivocation proof on reporting a validator.
"""
assert len(equivocated_block_hashes) >= 2
blocks = [
self.w3.eth.getBlock(block_hash)
for block_hash in equivocated_block_hashes
]
block_hashes_and_timestamp_strings = [
BLOCK_HASH_AND_TIMESTAMP_TEMPLATE.format(
block_hash=encode_hex(block.hash),
block_timestamp=datetime.datetime.utcfromtimestamp(
block.timestamp),
) for block in blocks
]
block_hash_and_timestamp_summary = "\n".join(
block_hashes_and_timestamp_strings)
# Use the first two blocks as representational data for the equivocation proof.
block_one = get_canonicalized_block(blocks[0])
block_two = get_canonicalized_block(blocks[1])
proposer_address_hex = encode_hex(get_proposer(block_one))
equivocation_report_template_variables = {
"proposer_address":
proposer_address_hex,
"block_step":
block_one.step,
"detection_time":
datetime.datetime.utcnow(),
"block_hash_timestamp_summary":
block_hash_and_timestamp_summary,
"rlp_encoded_block_header_one":
encode_hex(rlp_encoded_block(block_one)),
"signature_block_header_one":
keys.Signature(block_one.signature),
"rlp_encoded_block_header_two":
encode_hex(rlp_encoded_block(block_two)),
"signature_block_header_two":
keys.Signature(block_two.signature),
}
equivocation_report_file_name = (
f"equivocation_reports_for_proposer_{proposer_address_hex}")
with open(self.report_dir / equivocation_report_file_name,
"a") as equivocation_report_file:
equivocation_report_file.write(
EQUIVOCATION_REPORT_TEMPLATE.format(
**equivocation_report_template_variables))