def check(self, obj: FinishedSignagePointMessage) -> Optional[Event]:
if self._last_signage_point is None:
self._last_signage_point_timestamp = obj.timestamp
self._last_signage_point = obj.signage_point
return None
event = None
valid, skipped = calculate_skipped_signage_points(
self._last_signage_point_timestamp, self._last_signage_point,
obj.timestamp, obj.signage_point)
if not valid:
# Reset state when we receive non-valid order of signage points
# this ensures that we aren't sending any wrongly calculated skips
self._last_signage_point_timestamp = None
self._last_signage_point = None
return None
# To reduce notification spam, only send notifications for skips larger than 1
# or for multiple individual skips within 1 hour
if skipped == 1:
logging.info(f"Detected {skipped} skipped signage point.")
if self._last_skip_timestamp:
minutes_since_last_skip = (
datetime.now() - self._last_skip_timestamp).seconds // 60
if minutes_since_last_skip > 60:
logging.info(
"No other skips in the last 60 minutes. Can be safely ignored."
)
else:
message = "Experiencing networking issues? Skipped 2+ signage points in the last hour."
logging.warning(message)
event = Event(
type=EventType.USER,
priority=EventPriority.NORMAL,
service=EventService.FULL_NODE,
message=message,
)
if skipped >= 2:
message = f"Experiencing networking issues? Skipped {skipped} signage points!"
logging.warning(message)
event = Event(
type=EventType.USER,
priority=EventPriority.NORMAL,
service=EventService.FULL_NODE,
message=message,
)
if skipped != 0:
self._last_skip_timestamp = datetime.now()
self._last_signage_point_timestamp = obj.timestamp
self._last_signage_point = obj.signage_point
return event
def testShowcaseBadNotifications(self):
notifiers = [
PushoverNotifier(
title_prefix="Harvester 1",
config={
"enable": True,
"api_token": self.api_token,
"user_key": self.user_key
},
),
PushoverNotifier(
title_prefix="Harvester 2",
config={
"enable": True,
"api_token": self.api_token,
"user_key": self.user_key
},
),
PushoverNotifier(
title_prefix="Harvester 3",
config={
"enable": True,
"api_token": self.api_token,
"user_key": self.user_key
},
),
]
disconnected_hdd = Event(
type=EventType.USER,
priority=EventPriority.HIGH,
service=EventService.HARVESTER,
message=
"Disconnected HDD? The total plot count decreased from 101 to 42.",
)
network_issues = Event(
type=EventType.USER,
priority=EventPriority.NORMAL,
service=EventService.HARVESTER,
message=
"Experiencing networking issues? Harvester did not participate in any "
"challenge for 120 seconds. It's now working again.",
)
offline = Event(
type=EventType.USER,
priority=EventPriority.HIGH,
service=EventService.HARVESTER,
message=
"Your harvester appears to be offline! No events for the past 712 seconds.",
)
events = [disconnected_hdd, offline, network_issues]
for notifier, event in zip(notifiers, events):
success = notifier.send_events_to_user(events=[event])
self.assertTrue(success)
def get_low_priority_events():
return [
Event(
type=EventType.USER,
priority=EventPriority.LOW,
service=EventService.HARVESTER,
message="Low priority notification 1.",
),
Event(
type=EventType.USER,
priority=EventPriority.LOW,
service=EventService.HARVESTER,
message="Low priority notification 2.",
),
]
def check(self, obj: FinishedSignagePointMessage) -> Optional[Event]:
if self._last_signage_point is None:
self._last_signage_point_timestamp = obj.timestamp
self._last_signage_point = obj.signage_point
return None
event = None
valid, skipped = calculate_skipped_signage_points(
self._last_signage_point_timestamp, self._last_signage_point,
obj.timestamp, obj.signage_point)
if not valid:
return None
# To reduce notification spam, only send notifications for skips larger than 1
if skipped == 1:
logging.info(
f"Detected {skipped} skipped signage point."
"This is expected to happen occasionally and not a reason for concern."
)
elif skipped >= 2:
message = f"Experiencing networking issues? Skipped {skipped} signage points!"
logging.warning(message)
event = Event(
type=EventType.USER,
priority=EventPriority.NORMAL,
service=EventService.FULL_NODE,
message=message,
)
self._last_signage_point_timestamp = obj.timestamp
self._last_signage_point = obj.signage_point
return event
def testTelegramHighPriorityNotifications(self):
errors = self.notifier.send_events_to_user(events=[
Event(
type=EventType.USER,
priority=EventPriority.HIGH,
service=EventService.HARVESTER,
message="High priority notification 1.",
),
Event(
type=EventType.USER,
priority=EventPriority.HIGH,
service=EventService.HARVESTER,
message="High priority notification 2.",
),
])
self.assertFalse(errors)
def handle(self,
logs: str,
stats_manager: Optional[StatsManager] = None) -> List[Event]:
events = []
added_coin_messages = self._parser.parse(logs)
if stats_manager:
stats_manager.consume_wallet_messages(added_coin_messages)
total_mojos = 0
for coin_msg in added_coin_messages:
logging.info(
f"Cha-ching! Just received {coin_msg.amount_mojos} mojos.")
total_mojos += coin_msg.amount_mojos
if total_mojos > 0:
chia_coins = total_mojos / 1e12
xch_string = f"{chia_coins:.12f}".rstrip("0").rstrip(".")
events.append(
Event(
type=EventType.USER,
priority=EventPriority.LOW,
service=EventService.WALLET,
message=f"Cha-ching! Just received {xch_string} XCH ☘️",
))
return events
def handle(self,
logs: str,
stats_manager: Optional[StatsManager] = None) -> List[Event]:
"""Process incoming logs, check all conditions
and return a list of notable events.
"""
events = []
activity_messages = self._parser.parse(logs)
if stats_manager:
stats_manager.consume_harvester_messages(activity_messages)
# Create a keep-alive event if any logs indicating
# activity have been successfully parsed
if len(activity_messages) > 0:
logging.debug(f"Parsed {len(activity_messages)} activity messages")
events.append(
Event(type=EventType.KEEPALIVE,
priority=EventPriority.NORMAL,
service=EventService.HARVESTER,
message=""))
# Run messages through all condition checkers
for msg in activity_messages:
for checker in self._cond_checkers:
event = checker.check(msg)
if event:
events.append(event)
return events
def check(self, obj: HarvesterActivityMessage) -> Optional[Event]:
if self._last_timestamp is None:
self._last_timestamp = obj.timestamp
return None
event = None
seconds_since_last = (obj.timestamp - self._last_timestamp).seconds
if seconds_since_last > self._warning_threshold:
message = (
f"Experiencing networking issues? Harvester did not participate in any challenge "
f"for {seconds_since_last} seconds. It's now working again."
)
logging.warning(message)
event = Event(
type=EventType.USER, priority=EventPriority.NORMAL, service=EventService.HARVESTER, message=message
)
elif seconds_since_last > self._info_threshold:
# This threshold seems to be surpassed multiple times per day
# on the current network. So it only generates an INFO log.
logging.info(
f"Last farming event was {seconds_since_last} seconds ago."
" Usually every 9-10 seconds. No reason to worry if it happens "
" up to 20 times daily."
)
self._last_timestamp = obj.timestamp
return event
def testLowPrioriyNotifications(self):
errors = self.notifier.send_events_to_user(events=[
Event(
type=EventType.USER,
priority=EventPriority.LOW,
service=EventService.HARVESTER,
message="Low priority notification 1.",
),
Event(
type=EventType.USER,
priority=EventPriority.LOW,
service=EventService.HARVESTER,
message="Low priority notification 2.",
),
])
self.assertFalse(errors)
def check(self, obj: HarvesterActivityMessage) -> Optional[Event]:
if obj.total_plots_count > self._max_farmed_plots:
logging.info(
f"Detected new plots. Farming with {obj.total_plots_count} plots."
)
self._max_farmed_plots = obj.total_plots_count
event = None
if obj.total_plots_count < self._max_farmed_plots:
if self._max_farmed_plots - obj.total_plots_count < self._decrease_warn_threshold:
logging.info(
f"Plots decreased from {self._max_farmed_plots} to {obj.total_plots_count}. "
f"This is ignored because it's below threshold of {self._decrease_warn_threshold}"
)
else:
message = (
f"Disconnected HDD? The total plot count decreased from "
f"{self._max_farmed_plots} to {obj.total_plots_count}.")
logging.warning(message)
event = Event(type=EventType.USER,
priority=EventPriority.HIGH,
service=EventService.HARVESTER,
message=message)
# Update max plots to prevent repeated alarms
self._max_farmed_plots = obj.total_plots_count
return event
def get_normal_priority_events():
return [
Event(
type=EventType.USER,
priority=EventPriority.NORMAL,
service=EventService.HARVESTER,
message="Normal priority notification 1.",
),
]
def get_high_priority_events():
return [
Event(
type=EventType.USER,
priority=EventPriority.HIGH,
service=EventService.HARVESTER,
message="High priority notification 1.",
),
]
def check(self, obj: HarvesterActivityMessage) -> Optional[Event]:
if obj.found_proofs_count > 0:
message = f"Found {obj.found_proofs_count} proof(s)!"
logging.info(message)
return Event(type=EventType.USER,
priority=EventPriority.LOW,
service=EventService.HARVESTER,
message=message)
return None
def testHighPriorityNotifications(self):
errors = self.notifier.send_events_to_user(events=[
Event(
type=EventType.USER,
priority=EventPriority.HIGH,
service=EventService.HARVESTER,
message="This is a high priority notification!",
)
])
self.assertFalse(errors)
def testNormalPriorityNotifications(self):
errors = self.notifier.send_events_to_user(events=[
Event(
type=EventType.USER,
priority=EventPriority.NORMAL,
service=EventService.HARVESTER,
message="Normal priority notification.",
)
])
self.assertFalse(errors)
def check(self, obj: HarvesterActivityMessage) -> Optional[Event]:
if obj.search_time_seconds > self._warning_threshold:
message = f"Seeking plots took too long: {obj.search_time_seconds} seconds!"
logging.warning(message)
return Event(type=EventType.USER,
priority=EventPriority.NORMAL,
service=EventService.HARVESTER,
message=message)
return None
def _send_daily_notification(self):
summary = f"Hello farmer! 👋 Here's what happened in the last {self._frequency_hours} hours:\n"
for stat_acc in self._stat_accumulators:
summary += "\n" + stat_acc.get_summary()
stat_acc.reset()
self._notify_manager.process_events([
Event(type=EventType.DAILY_STATS,
priority=EventPriority.LOW,
service=EventService.DAILY,
message=summary)
])
def check(self, obj: FinishedSignagePointMessage) -> Optional[Event]:
if self._last_signage_point is None:
self._last_signage_point_timestamp = obj.timestamp
self._last_signage_point = obj.signage_point
return None
event = None
time_diff_seconds = (obj.timestamp -
self._last_signage_point_timestamp).seconds
increment_diff = obj.signage_point - (self._last_signage_point %
self._roll_over_point)
if increment_diff <= 0 or increment_diff > 1:
# This is hacky addition to prevent false alarms for some network-wide issues that
# aren't necessarily related to the local node. See "testNetworkScramble" test case.
# Signage points are expected approx every 8-10 seconds. If a point was skipped for real
# then we expect the time difference to be at least 2*8 seconds. Otherwise it's flaky event.
if time_diff_seconds < 15:
logging.info(
f"Detected unusual network activity. Last signage point {self._last_signage_point}, "
f"current signage point {obj.signage_point}. Time difference: {time_diff_seconds} "
f"seconds. Seems unrelated to the local node. Ignoring...")
else:
message = (
f"Experiencing networking issues? Skipped some signage points! "
f"Last {self._last_signage_point}/64, current {obj.signage_point}/64."
)
logging.warning(message)
event = Event(type=EventType.USER,
priority=EventPriority.NORMAL,
service=EventService.FULL_NODE,
message=message)
self._last_signage_point_timestamp = obj.timestamp
self._last_signage_point = obj.signage_point
return event
def testShowcaseGoodNotifications(self):
notifiers = [
PushoverNotifier(
title_prefix="Harvester 1",
config={"enable": True, "api_token": self.api_token, "user_key": self.user_key},
),
PushoverNotifier(
title_prefix="Harvester 2",
config={"enable": True, "api_token": self.api_token, "user_key": self.user_key},
),
PushoverNotifier(
title_prefix="Harvester 3",
config={"enable": True, "api_token": self.api_token, "user_key": self.user_key},
),
]
found_proof_event = Event(
type=EventType.USER,
priority=EventPriority.LOW,
service=EventService.HARVESTER,
message="Found 1 proof(s)!",
)
for notifier in notifiers:
errors = notifier.send_events_to_user(events=[found_proof_event])
self.assertFalse(errors)
def setUp(self) -> None:
self.threshold_seconds = 3
self.keep_alive_monitor = KeepAliveMonitor(thresholds={EventService.HARVESTER: self.threshold_seconds})
self.keep_alive_event = Event(
type=EventType.KEEPALIVE, priority=EventPriority.NORMAL, service=EventService.HARVESTER, message=""
)