def _get_db_conn_state():
conn_state = monitors.get_monitors([
MONITORS[monitor_names.database_connections],
MONITORS[monitor_names.database_connection_info]
])
return conn_state, False
def _get_query_insights():
query_insights = monitors.get_monitors([
MONITORS[monitor_names.frequent_queries],
MONITORS[monitor_names.slow_queries],
])
return query_insights, False
def get_health(args, use_redis_cache=True):
"""
Gets health status for the service
:param args: dictionary
:param args.verbose: bool
if True, returns db connection information
:param args.healthy_block_diff: int
determines the point at which a block difference is considered unhealthy
:param args.enforce_block_diff: bool
if true and the block difference is unhealthy an error is returned
:rtype: (dictionary, bool)
:return: tuple of health results and a boolean indicating an error
"""
redis = redis_connection.get_redis()
web3 = web3_provider.get_web3()
verbose = args.get("verbose")
enforce_block_diff = args.get("enforce_block_diff")
qs_healthy_block_diff = args.get("healthy_block_diff")
# If healthy block diff is given in url and positive, override config value
healthy_block_diff = qs_healthy_block_diff if qs_healthy_block_diff is not None \
and qs_healthy_block_diff >= 0 else default_healthy_block_diff
latest_block_num = None
latest_block_hash = None
latest_indexed_block_num = None
latest_indexed_block_hash = None
if use_redis_cache:
# get latest blockchain state from redis cache, or fallback to chain if None
latest_block_num, latest_block_hash = get_latest_chain_block_set_if_nx(
redis, web3)
# get latest db state from redis cache
latest_indexed_block_num = redis.get(
most_recent_indexed_block_redis_key)
if latest_indexed_block_num is not None:
latest_indexed_block_num = int(latest_indexed_block_num)
latest_indexed_block_hash = redis.get(
most_recent_indexed_block_hash_redis_key)
if latest_indexed_block_hash is not None:
latest_indexed_block_hash = latest_indexed_block_hash.decode(
"utf-8")
# fetch latest blockchain state from web3 if:
# we explicitly don't want to use redis cache or
# value from redis cache is None
if not use_redis_cache or latest_block_num is None or latest_block_hash is None:
# get latest blockchain state from web3
latest_block = web3.eth.getBlock("latest", True)
latest_block_num = latest_block.number
latest_block_hash = latest_block.hash.hex()
# fetch latest db state if:
# we explicitly don't want to use redis cache or
# value from redis cache is None
if not use_redis_cache or latest_indexed_block_num is None or latest_indexed_block_hash is None:
db_block_state = _get_db_block_state()
latest_indexed_block_num = db_block_state["number"] or 0
latest_indexed_block_hash = db_block_state["blockhash"]
trending_tracks_age_sec = get_elapsed_time_redis(
redis, trending_tracks_last_completion_redis_key)
trending_playlists_age_sec = get_elapsed_time_redis(
redis, trending_playlists_last_completion_redis_key)
# Get system information monitor values
sys_info = monitors.get_monitors([
MONITORS[monitor_names.database_size],
MONITORS[monitor_names.database_connections],
MONITORS[monitor_names.total_memory],
MONITORS[monitor_names.used_memory],
MONITORS[monitor_names.filesystem_size],
MONITORS[monitor_names.filesystem_used],
MONITORS[monitor_names.received_bytes_per_sec],
MONITORS[monitor_names.transferred_bytes_per_sec],
MONITORS[monitor_names.redis_total_memory]
])
health_results = {
"web": {
"blocknumber": latest_block_num,
"blockhash": latest_block_hash,
},
"db": {
"number": latest_indexed_block_num,
"blockhash": latest_indexed_block_hash
},
"git": os.getenv("GIT_SHA"),
"trending_tracks_age_sec": trending_tracks_age_sec,
"trending_playlists_age_sec": trending_playlists_age_sec,
"number_of_cpus": number_of_cpus,
**sys_info
}
block_difference = abs(health_results["web"]["blocknumber"] -
health_results["db"]["number"])
health_results["block_difference"] = block_difference
health_results[
"maximum_healthy_block_difference"] = default_healthy_block_diff
health_results.update(disc_prov_version)
if verbose:
# DB connections check
db_connections_json, error = _get_db_conn_state()
health_results["db_connections"] = db_connections_json
if error:
return health_results, error
# Return error on unhealthy block diff if requested.
if enforce_block_diff and health_results[
"block_difference"] > healthy_block_diff:
return health_results, True
return health_results, False
def get_health(args: GetHealthArgs,
use_redis_cache: bool = True) -> Tuple[Dict, bool]:
"""
Gets health status for the service
Returns a tuple of health results and a boolean indicating an error
"""
redis = redis_connection.get_redis()
web3 = web3_provider.get_web3()
verbose = args.get("verbose")
enforce_block_diff = args.get("enforce_block_diff")
qs_healthy_block_diff = cast(Optional[int], args.get("healthy_block_diff"))
challenge_events_age_max_drift = args.get("challenge_events_age_max_drift")
plays_count_max_drift = args.get("plays_count_max_drift")
# If healthy block diff is given in url and positive, override config value
healthy_block_diff = (
qs_healthy_block_diff if qs_healthy_block_diff is not None
and qs_healthy_block_diff >= 0 else default_healthy_block_diff)
latest_block_num: Optional[int] = None
latest_block_hash: Optional[str] = None
latest_indexed_block_num: Optional[int] = None
latest_indexed_block_hash: Optional[str] = None
if use_redis_cache:
# get latest blockchain state from redis cache, or fallback to chain if None
latest_block_num, latest_block_hash = get_latest_chain_block_set_if_nx(
redis, web3)
# get latest db state from redis cache
latest_indexed_block_num = redis.get(
most_recent_indexed_block_redis_key)
if latest_indexed_block_num is not None:
latest_indexed_block_num = int(latest_indexed_block_num)
latest_indexed_block_hash_bytes = redis.get(
most_recent_indexed_block_hash_redis_key)
if latest_indexed_block_hash_bytes is not None:
latest_indexed_block_hash = latest_indexed_block_hash_bytes.decode(
"utf-8")
else:
# Get latest blockchain state from web3
try:
latest_block = web3.eth.get_block("latest", True)
latest_block_num = latest_block.number
latest_block_hash = latest_block.hash.hex()
except Exception as e:
logger.error(f"Could not get latest block from chain: {e}")
# fetch latest db state if:
# we explicitly don't want to use redis cache or
# value from redis cache is None
if (not use_redis_cache or latest_indexed_block_num is None
or latest_indexed_block_hash is None):
db_block_state = _get_db_block_state()
latest_indexed_block_num = db_block_state["number"] or 0
latest_indexed_block_hash = db_block_state["blockhash"]
play_health_info = get_play_health_info(redis, plays_count_max_drift)
rewards_manager_health_info = get_rewards_manager_health_info(redis)
user_bank_health_info = get_user_bank_health_info(redis)
spl_audio_info = get_spl_audio_info(redis)
reactions_health_info = get_reactions_health_info(
redis,
args.get("reactions_max_indexing_drift"),
args.get("reactions_max_last_reaction_drift"),
)
trending_tracks_age_sec = get_elapsed_time_redis(
redis, trending_tracks_last_completion_redis_key)
trending_playlists_age_sec = get_elapsed_time_redis(
redis, trending_playlists_last_completion_redis_key)
challenge_events_age_sec = get_elapsed_time_redis(
redis, challenges_last_processed_event_redis_key)
user_balances_age_sec = get_elapsed_time_redis(
redis, user_balances_refresh_last_completion_redis_key)
num_users_in_lazy_balance_refresh_queue = int(
redis.scard(LAZY_REFRESH_REDIS_PREFIX))
num_users_in_immediate_balance_refresh_queue = int(
redis.scard(IMMEDIATE_REFRESH_REDIS_PREFIX))
last_scanned_block_for_balance_refresh = redis_get_or_restore(
redis, eth_indexing_last_scanned_block_key)
index_eth_age_sec = get_elapsed_time_redis(
redis, index_eth_last_completion_redis_key)
last_scanned_block_for_balance_refresh = (
int(last_scanned_block_for_balance_refresh)
if last_scanned_block_for_balance_refresh else None)
# Get system information monitor values
sys_info = monitors.get_monitors([
MONITORS[monitor_names.database_size],
MONITORS[monitor_names.database_connections],
MONITORS[monitor_names.total_memory],
MONITORS[monitor_names.used_memory],
MONITORS[monitor_names.filesystem_size],
MONITORS[monitor_names.filesystem_used],
MONITORS[monitor_names.received_bytes_per_sec],
MONITORS[monitor_names.transferred_bytes_per_sec],
MONITORS[monitor_names.redis_total_memory],
])
health_results = {
"web": {
"blocknumber": latest_block_num,
"blockhash": latest_block_hash,
},
"db": {
"number": latest_indexed_block_num,
"blockhash": latest_indexed_block_hash,
},
"git": os.getenv("GIT_SHA"),
"trending_tracks_age_sec": trending_tracks_age_sec,
"trending_playlists_age_sec": trending_playlists_age_sec,
"challenge_last_event_age_sec": challenge_events_age_sec,
"user_balances_age_sec": user_balances_age_sec,
"num_users_in_lazy_balance_refresh_queue":
num_users_in_lazy_balance_refresh_queue,
"num_users_in_immediate_balance_refresh_queue":
num_users_in_immediate_balance_refresh_queue,
"last_scanned_block_for_balance_refresh":
last_scanned_block_for_balance_refresh,
"index_eth_age_sec": index_eth_age_sec,
"number_of_cpus": number_of_cpus,
**sys_info,
"plays": play_health_info,
"rewards_manager": rewards_manager_health_info,
"user_bank": user_bank_health_info,
"openresty_public_key": openresty_public_key,
"spl_audio_info": spl_audio_info,
"reactions": reactions_health_info,
"infra_setup": infra_setup,
}
if latest_block_num is not None and latest_indexed_block_num is not None:
block_difference = abs(latest_block_num - latest_indexed_block_num)
else:
# If we cannot get a reading from chain about what the latest block is,
# we set the difference to be an unhealthy amount
block_difference = default_healthy_block_diff + 1
health_results["block_difference"] = block_difference
health_results[
"maximum_healthy_block_difference"] = default_healthy_block_diff
health_results.update(disc_prov_version)
# Check that this node meets the minimum system requirements
num_cpus: int = cast(int, health_results["number_of_cpus"] or 0)
total_memory: int = cast(int, health_results["total_memory"] or 0)
filesystem_size: int = cast(int, health_results["filesystem_size"] or 0)
if (num_cpus < min_number_of_cpus or total_memory < min_total_memory
or filesystem_size < min_filesystem_size):
health_results["meets_min_requirements"] = False
# TODO - this will become strictly enforced in upcoming service versions and return with error
else:
health_results["meets_min_requirements"] = True
if verbose:
# Elasticsearch health
if esclient:
health_results["elasticsearch"] = get_elasticsearch_health_info(
esclient, latest_indexed_block_num)
# DB connections check
db_connections_json, db_connections_error = _get_db_conn_state()
health_results["db_connections"] = db_connections_json
location = get_location()
health_results.update(location)
if db_connections_error:
return health_results, db_connections_error
query_insights_json, query_insights_error = _get_query_insights()
health_results["query_insights"] = query_insights_json
if query_insights_error:
return health_results, query_insights_error
table_size_info_json = monitors.get_monitors([
MONITORS[monitor_names.table_size_info],
])
health_results["tables"] = table_size_info_json
unhealthy_blocks = bool(enforce_block_diff
and block_difference > healthy_block_diff)
unhealthy_challenges = bool(
challenge_events_age_max_drift and challenge_events_age_sec
and challenge_events_age_sec > challenge_events_age_max_drift)
is_unhealthy = (unhealthy_blocks or unhealthy_challenges
or play_health_info["is_unhealthy"]
or reactions_health_info["is_unhealthy"])
return health_results, is_unhealthy
def get_health(args, use_redis_cache=True):
"""
Gets health status for the service
:param args: dictionary
:param args.verbose: bool
if True, returns db connection information
:param args.healthy_block_diff: int
determines the point at which a block difference is considered unhealthy
:param args.enforce_block_diff: bool
if true and the block difference is unhealthy an error is returned
:rtype: (dictionary, bool)
:return: tuple of health results and a boolean indicating an error
"""
redis = redis_connection.get_redis()
web3 = web3_provider.get_web3()
verbose = args.get("verbose")
enforce_block_diff = args.get("enforce_block_diff")
qs_healthy_block_diff = args.get("healthy_block_diff")
# If healthy block diff is given in url and positive, override config value
healthy_block_diff = qs_healthy_block_diff if qs_healthy_block_diff is not None \
and qs_healthy_block_diff >= 0 else default_healthy_block_diff
latest_block_num = None
latest_block_hash = None
# Get latest web block info
if use_redis_cache:
stored_latest_block_num = redis.get(latest_block_redis_key)
if stored_latest_block_num is not None:
latest_block_num = int(stored_latest_block_num)
stored_latest_blockhash = redis.get(latest_block_hash_redis_key)
if stored_latest_blockhash is not None:
latest_block_hash = stored_latest_blockhash.decode("utf-8")
if latest_block_num is None or latest_block_hash is None:
latest_block = web3.eth.getBlock("latest", True)
latest_block_num = latest_block.number
latest_block_hash = latest_block.hash.hex()
latest_indexed_block_num = None
latest_indexed_block_hash = None
# Get latest indexed block info
if use_redis_cache:
latest_indexed_block_num = redis.get(
most_recent_indexed_block_redis_key)
if latest_indexed_block_num is not None:
latest_indexed_block_num = int(latest_indexed_block_num)
latest_indexed_block_hash = redis.get(
most_recent_indexed_block_hash_redis_key)
if latest_indexed_block_hash is not None:
latest_indexed_block_hash = latest_indexed_block_hash.decode(
"utf-8")
if latest_indexed_block_num is None or latest_indexed_block_hash is None:
db_block_state = _get_db_block_state()
latest_indexed_block_num = db_block_state["number"] or 0
latest_indexed_block_hash = db_block_state["blockhash"]
# Get system information monitor values
sys_info = monitors.get_monitors([
MONITORS[monitor_names.database_size],
MONITORS[monitor_names.database_connections],
MONITORS[monitor_names.total_memory],
MONITORS[monitor_names.used_memory],
MONITORS[monitor_names.filesystem_size],
MONITORS[monitor_names.filesystem_used],
MONITORS[monitor_names.received_bytes_per_sec],
MONITORS[monitor_names.transferred_bytes_per_sec],
MONITORS[monitor_names.redis_total_memory]
])
health_results = {
"web": {
"blocknumber": latest_block_num,
"blockhash": latest_block_hash,
},
"db": {
"number": latest_indexed_block_num,
"blockhash": latest_indexed_block_hash
},
"git": os.getenv("GIT_SHA"),
**sys_info
}
block_difference = abs(health_results["web"]["blocknumber"] -
health_results["db"]["number"])
health_results["block_difference"] = block_difference
health_results[
"maximum_healthy_block_difference"] = default_healthy_block_diff
health_results.update(disc_prov_version)
if verbose:
# DB connections check
db_connections_json, error = _get_db_conn_state()
health_results["db_connections"] = db_connections_json
if error:
return health_results, error
# Return error on unhealthy block diff if requested.
if enforce_block_diff and health_results[
"block_difference"] > healthy_block_diff:
return health_results, True
return health_results, False
def get_celery_tasks():
celery_tasks = monitors.get_monitors([
MONITORS[monitor_names.celery_tasks],
])
return celery_tasks