def test_single_skip(primary_oracle, report_callback):
skip_reporter = SkipReporter(
state=SkipReporter.get_fresh_state(),
primary_oracle=primary_oracle,
grace_period=5,
)
skip_reporter.register_report_callback(report_callback)
# normal mode
for step in range(1, 21):
skip_reporter(mock_block(step, number=step))
# step 21 is skipped
# single skip should not be reported during grace period
for step in range(22, 27):
skip_reporter(mock_block(step, number=step - 1))
report_callback.assert_not_called()
# skip is reported after grace period is over
skip_reporter(mock_block(27, 26))
report_callback.assert_called_once_with(
primary_oracle.get_primary(height=21, step=21), SkippedProposal(21, 21)
)
report_callback.reset_mock()
# skip is not reported again
for step in range(28, 100):
skip_reporter(mock_block(step, step - 1))
report_callback.assert_not_called()
def test_skip_recovery(primary_oracle, report_callback):
skip_reporter = SkipReporter(
state=SkipReporter.get_fresh_state(),
primary_oracle=primary_oracle,
grace_period=5,
)
skip_reporter.register_report_callback(report_callback)
# normal until step 2
for step in range(1, 3):
skip_reporter(mock_block(step, number=step))
# validator 0 skips step 3
# normal until step 3 + grace_period
for step in range(4, 8):
skip_reporter(mock_block(step, number=step - 1))
# late proposal
skip_reporter(mock_block(3, number=3))
report_callback.assert_not_called()
# normal from then on
for step in range(4, 100):
skip_reporter(mock_block(step, number=step))
report_callback.assert_not_called()
def test_no_skips(report_callback, primary_oracle):
skip_reporter = SkipReporter(
state=SkipReporter.get_fresh_state(),
primary_oracle=primary_oracle,
grace_period=5,
)
skip_reporter.register_report_callback(report_callback)
for step in range(1, 100):
skip_reporter(mock_block(step, number=step))
report_callback.assert_not_called()
def test_do_not_report_skips_after_genesis(report_callback, primary_oracle):
# The genesis is always 0, so we do not want to report misses right after genesis
skip_reporter = SkipReporter(
state=SkipReporter.get_fresh_state(),
primary_oracle=primary_oracle,
grace_period=5,
)
skip_reporter.register_report_callback(report_callback)
for step in [0, 10, 11]:
skip_reporter(mock_block(step, number=step))
report_callback.assert_not_called()
def _initialize_app_state(self):
self.logger.info("no state entry found, starting from fresh state")
return AppStateV2(
block_fetcher_state=BlockFetcher.get_fresh_state(),
skip_reporter_state=SkipReporter.get_fresh_state(),
offline_reporter_state=OfflineReporter.get_fresh_state(),
)
def test_validator_offline(primary_oracle, report_callback, validators):
skip_reporter = SkipReporter(
state=SkipReporter.get_fresh_state(),
primary_oracle=primary_oracle,
grace_period=3,
)
skip_reporter.register_report_callback(report_callback)
number = 1
for step in range(1, 11):
if primary_oracle.get_primary(height=0, step=step) != validators[0]:
skip_reporter(mock_block(step, number=number))
number += 1
assert report_callback.call_args_list == [
call(validators[0], SkippedProposal(step=3, block_height=3)),
call(validators[0], SkippedProposal(step=6, block_height=5)),
]
def test_no_repeated_report_after_restart(primary_oracle, report_callback):
skip_reporter = SkipReporter(
state=SkipReporter.get_fresh_state(),
primary_oracle=primary_oracle,
grace_period=5,
)
# mine some blocks
for step in range(1, 21):
skip_reporter(mock_block(step, number=step))
# step 21 is skipped
# mine until report
for step in range(22, 28):
skip_reporter(mock_block(step, number=step - 1))
# restart, mine blocks, and check that no additional report is created
restarted_skip_reporter = SkipReporter(
state=skip_reporter.state, primary_oracle=primary_oracle, grace_period=5
)
skip_reporter.register_report_callback(report_callback)
for step in range(28, 100):
restarted_skip_reporter(mock_block(step, number=step - 1))
report_callback.assert_not_called()
def test_report_after_restart(primary_oracle, report_callback):
skip_reporter = SkipReporter(
state=SkipReporter.get_fresh_state(),
primary_oracle=primary_oracle,
grace_period=5,
)
# online at 1
skip_reporter(mock_block(1, number=1))
# offline at step 2
# online from step 3 to 4
for step in range(3, 5):
skip_reporter(mock_block(step, number=step - 1))
# restart
restarted_skip_reporter = SkipReporter(
state=skip_reporter.state, primary_oracle=primary_oracle, grace_period=5
)
restarted_skip_reporter.register_report_callback(report_callback)
# no reports in steps 5 to 7
for step in range(5, 8):
restarted_skip_reporter(mock_block(step, number=step - 1))
report_callback.assert_not_called()
# report at step 8
restarted_skip_reporter(mock_block(8, number=7))
report_callback.assert_called_once_with(
primary_oracle.get_primary(height=2, step=2),
SkippedProposal(step=2, block_height=2),
)
def _initialize_reporters(self, app_state, skip_rate, offline_window_size):
if not isinstance(app_state, AppStateV2):
raise InvalidAppStateException()
self.block_fetcher = BlockFetcher(
state=app_state.block_fetcher_state,
w3=self.w3,
db=self.db,
max_reorg_depth=MAX_REORG_DEPTH,
initial_block_resolver=self.initial_block_resolver,
)
self.skip_reporter = SkipReporter(
state=app_state.skip_reporter_state,
primary_oracle=self.primary_oracle,
grace_period=GRACE_PERIOD,
)
self.offline_reporter = OfflineReporter(
state=app_state.offline_reporter_state,
primary_oracle=self.primary_oracle,
offline_window_size=offline_window_size,
allowed_skip_rate=skip_rate,
)
self.equivocation_reporter = EquivocationReporter(db=self.db)
class App:
logger = structlog.get_logger("monitor.main")
def __init__(
self,
*,
rpc_uri,
chain_spec_path,
report_dir,
db_path,
skip_rate,
offline_window_size,
initial_block_resolver,
upgrade_db=False,
watch_chain_spec=False,
):
self.report_dir = report_dir
self.skip_file = open(report_dir / SKIP_FILE_NAME, "a")
self.w3 = None
self.epoch_fetcher = None
self.primary_oracle = None
self.db = None
self.block_fetcher = None
self.skip_reporter = None
self.offline_reporter = None
self.equivocation_reporter = None
self.initial_block_resolver = initial_block_resolver
self.chain_spec_path = chain_spec_path
self.original_chain_spec = None
self.watch_chain_spec = watch_chain_spec
self._initialize_db(db_path)
self._initialize_w3(rpc_uri)
self.wait_for_node_fully_synced()
self._initialize_primary_oracle(chain_spec_path)
app_state = self._load_app_state()
if upgrade_db:
app_state = self._upgrade_app_state(app_state)
self._initialize_reporters(app_state, skip_rate, offline_window_size)
self._register_reporter_callbacks()
self._running = False
def wait_for_node_fully_synced(self):
def is_synced(node_status):
syncmsg = "still syncing" if node_status.is_syncing else "fully synced"
self.logger.info(f"node {syncmsg}:{node_status}")
return not node_status.is_syncing
node_status.wait_for_node_status(self.w3, is_synced)
def run(self) -> None:
self._running = True
try:
self.logger.info("starting sync")
while self._running:
self._run_cycle()
finally:
self.skip_file.close()
def _run_cycle(self) -> None:
self._update_epochs()
with self.db.persistent_session() as session:
number_of_new_blocks = self.block_fetcher.fetch_and_insert_new_blocks(
max_number_of_blocks=500,
max_block_height=self.epoch_fetcher.last_fetch_height,
)
self.db.store_pickled(APP_STATE_KEY, self.app_state)
self.skip_file.flush()
session.commit()
self.logger.info(
f"Syncing ({self.block_fetcher.get_sync_status():.0%})"
if self.block_fetcher.syncing else "Synced",
head=format_block(self.block_fetcher.head),
head_hash=self.block_fetcher.head.hash.hex(),
)
if number_of_new_blocks == 0:
time.sleep(BLOCK_FETCH_INTERVAL)
# check at the end of the cycle so that we quit immediately when the chain spec has
# changed
self._check_chain_spec()
def _check_chain_spec(self) -> None:
if not self.watch_chain_spec:
return
with self.chain_spec_path.open("r") as f:
try:
chain_spec = json.load(f)
except json.JSONDecodeError:
chain_spec_has_changed = True
else:
chain_spec_has_changed = chain_spec != self.original_chain_spec
if chain_spec_has_changed:
self.logger.info("Chain spec file has changed.")
self.stop()
def _update_epochs(self) -> None:
new_epochs = self.epoch_fetcher.fetch_new_epochs()
for epoch in new_epochs:
self.primary_oracle.add_epoch(epoch)
self.primary_oracle.max_height = self.epoch_fetcher.last_fetch_height
def stop(self):
self.logger.info(
"Stopping tlbc-monitor. This may take a long time, please be patient!"
)
self._running = False
@property
def app_state(self):
return AppStateV2(
block_fetcher_state=self.block_fetcher.state,
skip_reporter_state=self.skip_reporter.state,
offline_reporter_state=self.offline_reporter.state,
)
#
# Initialization
#
def _initialize_db(self, db_path):
db_url = SQLITE_URL_FORMAT.format(path=db_path)
engine = create_engine(db_url)
self.db = BlockDB(engine)
def _initialize_w3(self, rpc_uri):
self.w3 = Web3(HTTPProvider(rpc_uri))
# Inject custom middleware to improve the handling of connection
# problems with the RPC endpoint.
if "http_retry_request" in self.w3.middleware_onion:
self.w3.middleware_onion.replace(
"http_retry_request", http_retry_request_middleware_endlessly)
else:
self.w3.middleware_onion.add(
http_retry_request_middleware_endlessly)
def _initialize_primary_oracle(self, chain_spec_path: Path) -> None:
with chain_spec_path.open("r") as f:
chain_spec = json.load(f)
self.original_chain_spec = chain_spec
validator_definition = chain_spec["engine"]["authorityRound"][
"params"]["validators"]
validator_definition_ranges = get_validator_definition_ranges(
validator_definition)
self.epoch_fetcher = EpochFetcher(self.w3,
validator_definition_ranges)
self.primary_oracle = PrimaryOracle()
static_epochs = get_static_epochs(validator_definition_ranges)
for epoch in static_epochs:
self.primary_oracle.add_epoch(epoch)
self._update_epochs()
def _initialize_reporters(self, app_state, skip_rate, offline_window_size):
if not isinstance(app_state, AppStateV2):
raise InvalidAppStateException()
self.block_fetcher = BlockFetcher(
state=app_state.block_fetcher_state,
w3=self.w3,
db=self.db,
max_reorg_depth=MAX_REORG_DEPTH,
initial_block_resolver=self.initial_block_resolver,
)
self.skip_reporter = SkipReporter(
state=app_state.skip_reporter_state,
primary_oracle=self.primary_oracle,
grace_period=GRACE_PERIOD,
)
self.offline_reporter = OfflineReporter(
state=app_state.offline_reporter_state,
primary_oracle=self.primary_oracle,
offline_window_size=offline_window_size,
allowed_skip_rate=skip_rate,
)
self.equivocation_reporter = EquivocationReporter(db=self.db)
def _initialize_app_state(self):
self.logger.info("no state entry found, starting from fresh state")
return AppStateV2(
block_fetcher_state=BlockFetcher.get_fresh_state(),
skip_reporter_state=SkipReporter.get_fresh_state(),
offline_reporter_state=OfflineReporter.get_fresh_state(),
)
def _load_app_state(self):
"""Loads and returns the app state object. Make sure do initialize the db first"""
return self.db.load_pickled(
APP_STATE_KEY) or self._initialize_app_state()
def _upgrade_app_state(self, app_state):
if isinstance(app_state, AppStateV1):
self.logger.info("Upgrade appstate from v1 to v2")
return upgrade_v1_to_v2(app_state)
elif isinstance(app_state, AppStateV2):
return app_state
else:
raise InvalidAppStateException(
"Can not upgrade unsupported app state version")
def _register_reporter_callbacks(self):
self.block_fetcher.register_report_callback(self.skip_reporter)
self.block_fetcher.register_report_callback(self.equivocation_reporter)
self.skip_reporter.register_report_callback(self.skip_logger)
self.skip_reporter.register_report_callback(self.offline_reporter)
self.offline_reporter.register_report_callback(self.offline_logger)
self.equivocation_reporter.register_report_callback(
self.equivocation_logger)
#
# Reporters
#
def skip_logger(self, validator, skipped_proposal):
skip_timestamp = step_number_to_timestamp(skipped_proposal.step)
self.skip_file.write("{},{},{}\n".format(
skipped_proposal.step,
encode_hex(validator),
datetime.datetime.utcfromtimestamp(skip_timestamp),
))
def offline_logger(self, validator, steps):
filename = (
f"offline_report_{encode_hex(validator)}_steps_{min(steps)}_to_{max(steps)}"
)
with open(self.report_dir / filename, "w") as f:
json.dump(
{
"validator": encode_hex(validator),
"missed_steps": steps
}, f)
def equivocation_logger(self, equivocated_block_hashes):
"""Log a reported equivocation event.
Equivocation reports are logged into files separated by the proposers
address. Logged information are the proposer of the blocks, the steps
at which all blocks have been equivocated and a list of all block hashes
with their timestamp. Additionally two representing blocks are logged
with their RLP encoded header and related signature, which can be used
for an equivocation proof on reporting a validator.
"""
assert len(equivocated_block_hashes) >= 2
blocks = [
self.w3.eth.getBlock(block_hash)
for block_hash in equivocated_block_hashes
]
block_hashes_and_timestamp_strings = [
BLOCK_HASH_AND_TIMESTAMP_TEMPLATE.format(
block_hash=encode_hex(block.hash),
block_timestamp=datetime.datetime.utcfromtimestamp(
block.timestamp),
) for block in blocks
]
block_hash_and_timestamp_summary = "\n".join(
block_hashes_and_timestamp_strings)
# Use the first two blocks as representational data for the equivocation proof.
block_one = get_canonicalized_block(blocks[0])
block_two = get_canonicalized_block(blocks[1])
proposer_address_hex = encode_hex(get_proposer(block_one))
equivocation_report_template_variables = {
"proposer_address":
proposer_address_hex,
"block_step":
block_one.step,
"detection_time":
datetime.datetime.utcnow(),
"block_hash_timestamp_summary":
block_hash_and_timestamp_summary,
"rlp_encoded_block_header_one":
encode_hex(rlp_encoded_block(block_one)),
"signature_block_header_one":
keys.Signature(block_one.signature),
"rlp_encoded_block_header_two":
encode_hex(rlp_encoded_block(block_two)),
"signature_block_header_two":
keys.Signature(block_two.signature),
}
equivocation_report_file_name = (
f"equivocation_reports_for_proposer_{proposer_address_hex}")
with open(self.report_dir / equivocation_report_file_name,
"a") as equivocation_report_file:
equivocation_report_file.write(
EQUIVOCATION_REPORT_TEMPLATE.format(
**equivocation_report_template_variables))