def _get_db_conn_state(): conn_state = monitors.get_monitors([ MONITORS[monitor_names.database_connections], MONITORS[monitor_names.database_connection_info] ]) return conn_state, False
def _get_query_insights(): query_insights = monitors.get_monitors([ MONITORS[monitor_names.frequent_queries], MONITORS[monitor_names.slow_queries], ]) return query_insights, False
def get_health(args, use_redis_cache=True): """ Gets health status for the service :param args: dictionary :param args.verbose: bool if True, returns db connection information :param args.healthy_block_diff: int determines the point at which a block difference is considered unhealthy :param args.enforce_block_diff: bool if true and the block difference is unhealthy an error is returned :rtype: (dictionary, bool) :return: tuple of health results and a boolean indicating an error """ redis = redis_connection.get_redis() web3 = web3_provider.get_web3() verbose = args.get("verbose") enforce_block_diff = args.get("enforce_block_diff") qs_healthy_block_diff = args.get("healthy_block_diff") # If healthy block diff is given in url and positive, override config value healthy_block_diff = qs_healthy_block_diff if qs_healthy_block_diff is not None \ and qs_healthy_block_diff >= 0 else default_healthy_block_diff latest_block_num = None latest_block_hash = None latest_indexed_block_num = None latest_indexed_block_hash = None if use_redis_cache: # get latest blockchain state from redis cache, or fallback to chain if None latest_block_num, latest_block_hash = get_latest_chain_block_set_if_nx( redis, web3) # get latest db state from redis cache latest_indexed_block_num = redis.get( most_recent_indexed_block_redis_key) if latest_indexed_block_num is not None: latest_indexed_block_num = int(latest_indexed_block_num) latest_indexed_block_hash = redis.get( most_recent_indexed_block_hash_redis_key) if latest_indexed_block_hash is not None: latest_indexed_block_hash = latest_indexed_block_hash.decode( "utf-8") # fetch latest blockchain state from web3 if: # we explicitly don't want to use redis cache or # value from redis cache is None if not use_redis_cache or latest_block_num is None or latest_block_hash is None: # get latest blockchain state from web3 latest_block = web3.eth.getBlock("latest", True) latest_block_num = latest_block.number latest_block_hash = latest_block.hash.hex() # fetch latest db state if: # we explicitly don't want to use redis cache or # value from redis cache is None if not use_redis_cache or latest_indexed_block_num is None or latest_indexed_block_hash is None: db_block_state = _get_db_block_state() latest_indexed_block_num = db_block_state["number"] or 0 latest_indexed_block_hash = db_block_state["blockhash"] trending_tracks_age_sec = get_elapsed_time_redis( redis, trending_tracks_last_completion_redis_key) trending_playlists_age_sec = get_elapsed_time_redis( redis, trending_playlists_last_completion_redis_key) # Get system information monitor values sys_info = monitors.get_monitors([ MONITORS[monitor_names.database_size], MONITORS[monitor_names.database_connections], MONITORS[monitor_names.total_memory], MONITORS[monitor_names.used_memory], MONITORS[monitor_names.filesystem_size], MONITORS[monitor_names.filesystem_used], MONITORS[monitor_names.received_bytes_per_sec], MONITORS[monitor_names.transferred_bytes_per_sec], MONITORS[monitor_names.redis_total_memory] ]) health_results = { "web": { "blocknumber": latest_block_num, "blockhash": latest_block_hash, }, "db": { "number": latest_indexed_block_num, "blockhash": latest_indexed_block_hash }, "git": os.getenv("GIT_SHA"), "trending_tracks_age_sec": trending_tracks_age_sec, "trending_playlists_age_sec": trending_playlists_age_sec, "number_of_cpus": number_of_cpus, **sys_info } block_difference = abs(health_results["web"]["blocknumber"] - health_results["db"]["number"]) health_results["block_difference"] = block_difference health_results[ "maximum_healthy_block_difference"] = default_healthy_block_diff health_results.update(disc_prov_version) if verbose: # DB connections check db_connections_json, error = _get_db_conn_state() health_results["db_connections"] = db_connections_json if error: return health_results, error # Return error on unhealthy block diff if requested. if enforce_block_diff and health_results[ "block_difference"] > healthy_block_diff: return health_results, True return health_results, False
def get_health(args: GetHealthArgs, use_redis_cache: bool = True) -> Tuple[Dict, bool]: """ Gets health status for the service Returns a tuple of health results and a boolean indicating an error """ redis = redis_connection.get_redis() web3 = web3_provider.get_web3() verbose = args.get("verbose") enforce_block_diff = args.get("enforce_block_diff") qs_healthy_block_diff = cast(Optional[int], args.get("healthy_block_diff")) challenge_events_age_max_drift = args.get("challenge_events_age_max_drift") plays_count_max_drift = args.get("plays_count_max_drift") # If healthy block diff is given in url and positive, override config value healthy_block_diff = ( qs_healthy_block_diff if qs_healthy_block_diff is not None and qs_healthy_block_diff >= 0 else default_healthy_block_diff) latest_block_num: Optional[int] = None latest_block_hash: Optional[str] = None latest_indexed_block_num: Optional[int] = None latest_indexed_block_hash: Optional[str] = None if use_redis_cache: # get latest blockchain state from redis cache, or fallback to chain if None latest_block_num, latest_block_hash = get_latest_chain_block_set_if_nx( redis, web3) # get latest db state from redis cache latest_indexed_block_num = redis.get( most_recent_indexed_block_redis_key) if latest_indexed_block_num is not None: latest_indexed_block_num = int(latest_indexed_block_num) latest_indexed_block_hash_bytes = redis.get( most_recent_indexed_block_hash_redis_key) if latest_indexed_block_hash_bytes is not None: latest_indexed_block_hash = latest_indexed_block_hash_bytes.decode( "utf-8") else: # Get latest blockchain state from web3 try: latest_block = web3.eth.get_block("latest", True) latest_block_num = latest_block.number latest_block_hash = latest_block.hash.hex() except Exception as e: logger.error(f"Could not get latest block from chain: {e}") # fetch latest db state if: # we explicitly don't want to use redis cache or # value from redis cache is None if (not use_redis_cache or latest_indexed_block_num is None or latest_indexed_block_hash is None): db_block_state = _get_db_block_state() latest_indexed_block_num = db_block_state["number"] or 0 latest_indexed_block_hash = db_block_state["blockhash"] play_health_info = get_play_health_info(redis, plays_count_max_drift) rewards_manager_health_info = get_rewards_manager_health_info(redis) user_bank_health_info = get_user_bank_health_info(redis) spl_audio_info = get_spl_audio_info(redis) reactions_health_info = get_reactions_health_info( redis, args.get("reactions_max_indexing_drift"), args.get("reactions_max_last_reaction_drift"), ) trending_tracks_age_sec = get_elapsed_time_redis( redis, trending_tracks_last_completion_redis_key) trending_playlists_age_sec = get_elapsed_time_redis( redis, trending_playlists_last_completion_redis_key) challenge_events_age_sec = get_elapsed_time_redis( redis, challenges_last_processed_event_redis_key) user_balances_age_sec = get_elapsed_time_redis( redis, user_balances_refresh_last_completion_redis_key) num_users_in_lazy_balance_refresh_queue = int( redis.scard(LAZY_REFRESH_REDIS_PREFIX)) num_users_in_immediate_balance_refresh_queue = int( redis.scard(IMMEDIATE_REFRESH_REDIS_PREFIX)) last_scanned_block_for_balance_refresh = redis_get_or_restore( redis, eth_indexing_last_scanned_block_key) index_eth_age_sec = get_elapsed_time_redis( redis, index_eth_last_completion_redis_key) last_scanned_block_for_balance_refresh = ( int(last_scanned_block_for_balance_refresh) if last_scanned_block_for_balance_refresh else None) # Get system information monitor values sys_info = monitors.get_monitors([ MONITORS[monitor_names.database_size], MONITORS[monitor_names.database_connections], MONITORS[monitor_names.total_memory], MONITORS[monitor_names.used_memory], MONITORS[monitor_names.filesystem_size], MONITORS[monitor_names.filesystem_used], MONITORS[monitor_names.received_bytes_per_sec], MONITORS[monitor_names.transferred_bytes_per_sec], MONITORS[monitor_names.redis_total_memory], ]) health_results = { "web": { "blocknumber": latest_block_num, "blockhash": latest_block_hash, }, "db": { "number": latest_indexed_block_num, "blockhash": latest_indexed_block_hash, }, "git": os.getenv("GIT_SHA"), "trending_tracks_age_sec": trending_tracks_age_sec, "trending_playlists_age_sec": trending_playlists_age_sec, "challenge_last_event_age_sec": challenge_events_age_sec, "user_balances_age_sec": user_balances_age_sec, "num_users_in_lazy_balance_refresh_queue": num_users_in_lazy_balance_refresh_queue, "num_users_in_immediate_balance_refresh_queue": num_users_in_immediate_balance_refresh_queue, "last_scanned_block_for_balance_refresh": last_scanned_block_for_balance_refresh, "index_eth_age_sec": index_eth_age_sec, "number_of_cpus": number_of_cpus, **sys_info, "plays": play_health_info, "rewards_manager": rewards_manager_health_info, "user_bank": user_bank_health_info, "openresty_public_key": openresty_public_key, "spl_audio_info": spl_audio_info, "reactions": reactions_health_info, "infra_setup": infra_setup, } if latest_block_num is not None and latest_indexed_block_num is not None: block_difference = abs(latest_block_num - latest_indexed_block_num) else: # If we cannot get a reading from chain about what the latest block is, # we set the difference to be an unhealthy amount block_difference = default_healthy_block_diff + 1 health_results["block_difference"] = block_difference health_results[ "maximum_healthy_block_difference"] = default_healthy_block_diff health_results.update(disc_prov_version) # Check that this node meets the minimum system requirements num_cpus: int = cast(int, health_results["number_of_cpus"] or 0) total_memory: int = cast(int, health_results["total_memory"] or 0) filesystem_size: int = cast(int, health_results["filesystem_size"] or 0) if (num_cpus < min_number_of_cpus or total_memory < min_total_memory or filesystem_size < min_filesystem_size): health_results["meets_min_requirements"] = False # TODO - this will become strictly enforced in upcoming service versions and return with error else: health_results["meets_min_requirements"] = True if verbose: # Elasticsearch health if esclient: health_results["elasticsearch"] = get_elasticsearch_health_info( esclient, latest_indexed_block_num) # DB connections check db_connections_json, db_connections_error = _get_db_conn_state() health_results["db_connections"] = db_connections_json location = get_location() health_results.update(location) if db_connections_error: return health_results, db_connections_error query_insights_json, query_insights_error = _get_query_insights() health_results["query_insights"] = query_insights_json if query_insights_error: return health_results, query_insights_error table_size_info_json = monitors.get_monitors([ MONITORS[monitor_names.table_size_info], ]) health_results["tables"] = table_size_info_json unhealthy_blocks = bool(enforce_block_diff and block_difference > healthy_block_diff) unhealthy_challenges = bool( challenge_events_age_max_drift and challenge_events_age_sec and challenge_events_age_sec > challenge_events_age_max_drift) is_unhealthy = (unhealthy_blocks or unhealthy_challenges or play_health_info["is_unhealthy"] or reactions_health_info["is_unhealthy"]) return health_results, is_unhealthy
def get_health(args, use_redis_cache=True): """ Gets health status for the service :param args: dictionary :param args.verbose: bool if True, returns db connection information :param args.healthy_block_diff: int determines the point at which a block difference is considered unhealthy :param args.enforce_block_diff: bool if true and the block difference is unhealthy an error is returned :rtype: (dictionary, bool) :return: tuple of health results and a boolean indicating an error """ redis = redis_connection.get_redis() web3 = web3_provider.get_web3() verbose = args.get("verbose") enforce_block_diff = args.get("enforce_block_diff") qs_healthy_block_diff = args.get("healthy_block_diff") # If healthy block diff is given in url and positive, override config value healthy_block_diff = qs_healthy_block_diff if qs_healthy_block_diff is not None \ and qs_healthy_block_diff >= 0 else default_healthy_block_diff latest_block_num = None latest_block_hash = None # Get latest web block info if use_redis_cache: stored_latest_block_num = redis.get(latest_block_redis_key) if stored_latest_block_num is not None: latest_block_num = int(stored_latest_block_num) stored_latest_blockhash = redis.get(latest_block_hash_redis_key) if stored_latest_blockhash is not None: latest_block_hash = stored_latest_blockhash.decode("utf-8") if latest_block_num is None or latest_block_hash is None: latest_block = web3.eth.getBlock("latest", True) latest_block_num = latest_block.number latest_block_hash = latest_block.hash.hex() latest_indexed_block_num = None latest_indexed_block_hash = None # Get latest indexed block info if use_redis_cache: latest_indexed_block_num = redis.get( most_recent_indexed_block_redis_key) if latest_indexed_block_num is not None: latest_indexed_block_num = int(latest_indexed_block_num) latest_indexed_block_hash = redis.get( most_recent_indexed_block_hash_redis_key) if latest_indexed_block_hash is not None: latest_indexed_block_hash = latest_indexed_block_hash.decode( "utf-8") if latest_indexed_block_num is None or latest_indexed_block_hash is None: db_block_state = _get_db_block_state() latest_indexed_block_num = db_block_state["number"] or 0 latest_indexed_block_hash = db_block_state["blockhash"] # Get system information monitor values sys_info = monitors.get_monitors([ MONITORS[monitor_names.database_size], MONITORS[monitor_names.database_connections], MONITORS[monitor_names.total_memory], MONITORS[monitor_names.used_memory], MONITORS[monitor_names.filesystem_size], MONITORS[monitor_names.filesystem_used], MONITORS[monitor_names.received_bytes_per_sec], MONITORS[monitor_names.transferred_bytes_per_sec], MONITORS[monitor_names.redis_total_memory] ]) health_results = { "web": { "blocknumber": latest_block_num, "blockhash": latest_block_hash, }, "db": { "number": latest_indexed_block_num, "blockhash": latest_indexed_block_hash }, "git": os.getenv("GIT_SHA"), **sys_info } block_difference = abs(health_results["web"]["blocknumber"] - health_results["db"]["number"]) health_results["block_difference"] = block_difference health_results[ "maximum_healthy_block_difference"] = default_healthy_block_diff health_results.update(disc_prov_version) if verbose: # DB connections check db_connections_json, error = _get_db_conn_state() health_results["db_connections"] = db_connections_json if error: return health_results, error # Return error on unhealthy block diff if requested. if enforce_block_diff and health_results[ "block_difference"] > healthy_block_diff: return health_results, True return health_results, False
def get_celery_tasks(): celery_tasks = monitors.get_monitors([ MONITORS[monitor_names.celery_tasks], ]) return celery_tasks