def check_thorchain_midgard_api(context, node_address):
"""
Check that Midgard API is ok
"""
chat_id = context.job.context['chat_id']
node_data = context.job.context['chat_data']['nodes'][node_address]
was_healthy = node_data.setdefault('is_midgard_healthy', True)
is_midgard_healthy = is_midgard_api_healthy(node_data['ip_address'])
if was_healthy != is_midgard_healthy:
if is_midgard_healthy:
text = 'Midgard API is healthy again! 👌' + '\n' + \
'IP: ' + node_data['ip_address'] + '\n' + \
'THORNode: ' + node_data['alias'] + '\n' + \
'Node address: ' + node_address
try_message_with_home_menu(context, chat_id=chat_id, text=text)
else:
text = 'Midgard API is not healthy anymore! 💀' + '\n' + \
'IP: ' + node_data['ip_address'] + '\n' + \
'THORNode: ' + node_data['alias'] + '\n' + \
'Node address: ' + node_address + '\n\n' + \
'Please check your Thornode immediately!'
try_message_with_home_menu(context, chat_id=chat_id, text=text)
node_data['is_midgard_healthy'] = is_midgard_healthy
def is_thornode_healthy(context, node_address) -> bool:
chat_id = context.job.context['chat_id']
node_data = context.job.context['chat_data']['nodes'][node_address]
# If not initialized assuming node was healhty.
if "healthy" not in context.job.context['chat_data']['nodes'][
node_address]:
context.job.context['chat_data']['nodes'][node_address][
"healthy"] = True
was_healthy = node_data["healthy"]
try:
get_latest_block_height(node_data['ip_address'])
if not was_healthy:
try_message_with_home_menu(
context=context,
chat_id=chat_id,
text=get_node_healthy_again_message(node_data))
context.job.context['chat_data']['nodes'][node_address][
"healthy"] = True
return True
except (Timeout, ConnectionError):
if was_healthy:
try_message_with_home_menu(
context=context,
chat_id=chat_id,
text=get_node_health_warning_message(node_data))
context.job.context['chat_data']['nodes'][node_address][
"healthy"] = False
return False
def check_versions_status(context):
chat_data = context.job.context['chat_data']
try:
node_accounts = get_node_accounts()
except Exception as e:
logger.exception(e)
logger.error(
"I couldn't get the node accounts while checking version status.")
return
highest_version = max(map(lambda n: n['version'], node_accounts),
key=lambda v: version.parse(v))
last_newest_version = chat_data.get('newest_software_version', None)
if last_newest_version is None or version.parse(
highest_version) > version.parse(last_newest_version):
chat_data['newest_software_version'] = highest_version
for node in chat_data.get('nodes', {}).values():
if version.parse(node['version']) < version.parse(highest_version):
message = f"Consider updating the software on your node: *{node['alias']}* ‼️\n" \
f"Your software version is *{node['version']}* " \
f"but one of the nodes already runs on *{highest_version}*"
try_message_with_home_menu(
context,
chat_id=context.job.context['chat_id'],
text=message)
def check_thorchain_block_height(context, node_address):
chat_id = context.job.context['chat_id']
node_data = context.job.context['chat_data']['nodes'][node_address]
try:
block_height = get_latest_block_height(node_data['ip_address'])
except (Timeout, ConnectionError):
logger.warning(f"Timeout or Connection error with {node_data['ip_address']}")
return
is_stuck = block_height <= node_data.setdefault('block_height', 0)
block_height_stuck_count = node_data.setdefault("block_height_stuck_count", 0)
if is_stuck:
block_height_stuck_count += 1
if block_height_stuck_count == 1:
text = 'Block height is not increasing anymore! 💀' + '\n' + \
'IP: ' + node_data['ip_address'] + '\n' + \
'THORNode: ' + node_data['alias'] + '\n' + \
'Node address: ' + node_address + '\n' + \
'Block height stuck at: ' + block_height + '\n\n' + \
'Please check your Thornode immediately!'
try_message_with_home_menu(context=context, chat_id=chat_id, text=text)
else:
if block_height_stuck_count >= 1:
text = f"Block height is increasing again! 👌\n" + \
f"IP: {node_data['ip_address']}\n" + \
f"THORNode: {node_data['alias']}\n" + \
f"Node address: {node_address}\n" + \
f"Block height now at: {block_height}\n"
try_message_with_home_menu(context=context, chat_id=chat_id, text=text)
block_height_stuck_count = 0
node_data['block_height'] = block_height
node_data["block_height_stuck_count"] = block_height_stuck_count
def check_thornodes(context):
chat_id = context.job.context['chat_id']
chat_data = context.job.context['chat_data']
inactive_nodes = []
for node_address, local_node in chat_data.get('nodes', {}).items():
try:
remote_node = get_thornode_object_or_none(address=node_address)
except HTTPError as e:
logger.exception(e)
continue
if remote_node is None:
text = 'THORNode ' + local_node['alias'] + ' is not active anymore! 💀' + '\n' + \
'Address: ' + node_address + '\n\n' + \
'Please enter another THORNode address.'
inactive_nodes.append(node_address)
try_message_with_home_menu(context=context,
chat_id=chat_id,
text=text)
continue
is_not_blocked = float(local_node['last_notification_timestamp']) < \
datetime.timestamp(
datetime.now() - timedelta(seconds=local_node['notification_timeout_in_seconds']))
if is_not_blocked:
message = build_notification_message_for_active_node(local_node, remote_node, context)
if message:
# Update data
local_node['status'] = remote_node['status']
local_node['bond'] = remote_node['bond']
local_node['slash_points'] = remote_node['slash_points']
local_node['ip_address'] = remote_node['ip_address']
local_node['last_notification_timestamp'] = datetime.timestamp(datetime.now())
local_node['notification_timeout_in_seconds'] *= NOTIFICATION_TIMEOUT_MULTIPLIER
try_message_with_home_menu(context=context,
chat_id=chat_id,
text=message)
else:
local_node['notification_timeout_in_seconds'] = INITIAL_NOTIFICATION_TIMEOUT
if local_node['status'].upper() in MONITORED_STATUSES and is_thornode_healthy(context, node_address):
check_thorchain_block_height(context, node_address=node_address)
check_thorchain_catch_up_status(context, node_address=node_address)
check_thorchain_midgard_api(context, node_address=node_address)
for node_address in inactive_nodes:
del chat_data['nodes'][node_address]
def check_thorchain_catch_up_status(context, node_address):
"""
Check if node is some blocks behind with catch up status
"""
chat_id = context.job.context['chat_id']
node_data = context.job.context['chat_data']['nodes'][node_address]
if 'is_catching_up' not in node_data:
node_data['is_catching_up'] = False
try:
is_currently_catching_up = is_thorchain_catching_up(
node_data['ip_address'])
except (Timeout, ConnectionError):
logger.warning(
f"Timeout or Connection error with {node_data['ip_address']}")
return
if node_data['is_catching_up'] != is_currently_catching_up:
try:
block_height = get_latest_block_height(node_data['ip_address'])
except (Timeout, ConnectionError):
logger.warning(
f"Timeout or Connection error with {node_data['ip_address']}")
block_height = "currently unavailable"
if is_currently_catching_up:
node_data['is_catching_up'] = True
text = 'The Node is behind the latest block height and catching up! 💀 ' + '\n' + \
'IP: ' + node_data['ip_address'] + '\n' + \
'THORNode: ' + node_data['alias'] + '\n' + \
'Node address: ' + node_address + '\n' + \
'Current block height: ' + block_height + '\n\n' + \
'Please check your Thornode immediately!'
else:
node_data['is_catching_up'] = False
text = 'The node caught up to the latest block height again! 👌' + '\n' + \
'IP: ' + node_data['ip_address'] + '\n' + \
'THORNode: ' + node_data['alias'] + '\n' + \
'Node address: ' + node_address + '\n' + \
'Current block height: ' + block_height
try_message_with_home_menu(context=context, chat_id=chat_id, text=text)