def check_thorchain_midgard_api(context, node_address): """ Check that Midgard API is ok """ chat_id = context.job.context['chat_id'] node_data = context.job.context['chat_data']['nodes'][node_address] was_healthy = node_data.setdefault('is_midgard_healthy', True) is_midgard_healthy = is_midgard_api_healthy(node_data['ip_address']) if was_healthy != is_midgard_healthy: if is_midgard_healthy: text = 'Midgard API is healthy again! ๐' + '\n' + \ 'IP: ' + node_data['ip_address'] + '\n' + \ 'THORNode: ' + node_data['alias'] + '\n' + \ 'Node address: ' + node_address try_message_with_home_menu(context, chat_id=chat_id, text=text) else: text = 'Midgard API is not healthy anymore! ๐' + '\n' + \ 'IP: ' + node_data['ip_address'] + '\n' + \ 'THORNode: ' + node_data['alias'] + '\n' + \ 'Node address: ' + node_address + '\n\n' + \ 'Please check your Thornode immediately!' try_message_with_home_menu(context, chat_id=chat_id, text=text) node_data['is_midgard_healthy'] = is_midgard_healthy
def is_thornode_healthy(context, node_address) -> bool: chat_id = context.job.context['chat_id'] node_data = context.job.context['chat_data']['nodes'][node_address] # If not initialized assuming node was healhty. if "healthy" not in context.job.context['chat_data']['nodes'][ node_address]: context.job.context['chat_data']['nodes'][node_address][ "healthy"] = True was_healthy = node_data["healthy"] try: get_latest_block_height(node_data['ip_address']) if not was_healthy: try_message_with_home_menu( context=context, chat_id=chat_id, text=get_node_healthy_again_message(node_data)) context.job.context['chat_data']['nodes'][node_address][ "healthy"] = True return True except (Timeout, ConnectionError): if was_healthy: try_message_with_home_menu( context=context, chat_id=chat_id, text=get_node_health_warning_message(node_data)) context.job.context['chat_data']['nodes'][node_address][ "healthy"] = False return False
def check_versions_status(context): chat_data = context.job.context['chat_data'] try: node_accounts = get_node_accounts() except Exception as e: logger.exception(e) logger.error( "I couldn't get the node accounts while checking version status.") return highest_version = max(map(lambda n: n['version'], node_accounts), key=lambda v: version.parse(v)) last_newest_version = chat_data.get('newest_software_version', None) if last_newest_version is None or version.parse( highest_version) > version.parse(last_newest_version): chat_data['newest_software_version'] = highest_version for node in chat_data.get('nodes', {}).values(): if version.parse(node['version']) < version.parse(highest_version): message = f"Consider updating the software on your node: *{node['alias']}* โผ๏ธ\n" \ f"Your software version is *{node['version']}* " \ f"but one of the nodes already runs on *{highest_version}*" try_message_with_home_menu( context, chat_id=context.job.context['chat_id'], text=message)
def check_thorchain_block_height(context, node_address): chat_id = context.job.context['chat_id'] node_data = context.job.context['chat_data']['nodes'][node_address] try: block_height = get_latest_block_height(node_data['ip_address']) except (Timeout, ConnectionError): logger.warning(f"Timeout or Connection error with {node_data['ip_address']}") return is_stuck = block_height <= node_data.setdefault('block_height', 0) block_height_stuck_count = node_data.setdefault("block_height_stuck_count", 0) if is_stuck: block_height_stuck_count += 1 if block_height_stuck_count == 1: text = 'Block height is not increasing anymore! ๐' + '\n' + \ 'IP: ' + node_data['ip_address'] + '\n' + \ 'THORNode: ' + node_data['alias'] + '\n' + \ 'Node address: ' + node_address + '\n' + \ 'Block height stuck at: ' + block_height + '\n\n' + \ 'Please check your Thornode immediately!' try_message_with_home_menu(context=context, chat_id=chat_id, text=text) else: if block_height_stuck_count >= 1: text = f"Block height is increasing again! ๐\n" + \ f"IP: {node_data['ip_address']}\n" + \ f"THORNode: {node_data['alias']}\n" + \ f"Node address: {node_address}\n" + \ f"Block height now at: {block_height}\n" try_message_with_home_menu(context=context, chat_id=chat_id, text=text) block_height_stuck_count = 0 node_data['block_height'] = block_height node_data["block_height_stuck_count"] = block_height_stuck_count
def check_thornodes(context): chat_id = context.job.context['chat_id'] chat_data = context.job.context['chat_data'] inactive_nodes = [] for node_address, local_node in chat_data.get('nodes', {}).items(): try: remote_node = get_thornode_object_or_none(address=node_address) except HTTPError as e: logger.exception(e) continue if remote_node is None: text = 'THORNode ' + local_node['alias'] + ' is not active anymore! ๐' + '\n' + \ 'Address: ' + node_address + '\n\n' + \ 'Please enter another THORNode address.' inactive_nodes.append(node_address) try_message_with_home_menu(context=context, chat_id=chat_id, text=text) continue is_not_blocked = float(local_node['last_notification_timestamp']) < \ datetime.timestamp( datetime.now() - timedelta(seconds=local_node['notification_timeout_in_seconds'])) if is_not_blocked: message = build_notification_message_for_active_node(local_node, remote_node, context) if message: # Update data local_node['status'] = remote_node['status'] local_node['bond'] = remote_node['bond'] local_node['slash_points'] = remote_node['slash_points'] local_node['ip_address'] = remote_node['ip_address'] local_node['last_notification_timestamp'] = datetime.timestamp(datetime.now()) local_node['notification_timeout_in_seconds'] *= NOTIFICATION_TIMEOUT_MULTIPLIER try_message_with_home_menu(context=context, chat_id=chat_id, text=message) else: local_node['notification_timeout_in_seconds'] = INITIAL_NOTIFICATION_TIMEOUT if local_node['status'].upper() in MONITORED_STATUSES and is_thornode_healthy(context, node_address): check_thorchain_block_height(context, node_address=node_address) check_thorchain_catch_up_status(context, node_address=node_address) check_thorchain_midgard_api(context, node_address=node_address) for node_address in inactive_nodes: del chat_data['nodes'][node_address]
def check_thorchain_catch_up_status(context, node_address): """ Check if node is some blocks behind with catch up status """ chat_id = context.job.context['chat_id'] node_data = context.job.context['chat_data']['nodes'][node_address] if 'is_catching_up' not in node_data: node_data['is_catching_up'] = False try: is_currently_catching_up = is_thorchain_catching_up( node_data['ip_address']) except (Timeout, ConnectionError): logger.warning( f"Timeout or Connection error with {node_data['ip_address']}") return if node_data['is_catching_up'] != is_currently_catching_up: try: block_height = get_latest_block_height(node_data['ip_address']) except (Timeout, ConnectionError): logger.warning( f"Timeout or Connection error with {node_data['ip_address']}") block_height = "currently unavailable" if is_currently_catching_up: node_data['is_catching_up'] = True text = 'The Node is behind the latest block height and catching up! ๐ ' + '\n' + \ 'IP: ' + node_data['ip_address'] + '\n' + \ 'THORNode: ' + node_data['alias'] + '\n' + \ 'Node address: ' + node_address + '\n' + \ 'Current block height: ' + block_height + '\n\n' + \ 'Please check your Thornode immediately!' else: node_data['is_catching_up'] = False text = 'The node caught up to the latest block height again! ๐' + '\n' + \ 'IP: ' + node_data['ip_address'] + '\n' + \ 'THORNode: ' + node_data['alias'] + '\n' + \ 'Node address: ' + node_address + '\n' + \ 'Current block height: ' + block_height try_message_with_home_menu(context=context, chat_id=chat_id, text=text)