def inner_wrap(*args, **kwargs):
message = request.headers.get(MESSAGE_HEADER)
signature = request.headers.get(SIGNATURE_HEADER)
authed_user_id = None
if message and signature:
web3 = web3_provider.get_web3()
encoded_to_recover = encode_defunct(text=message)
wallet = web3.eth.account.recover_message(
encoded_to_recover, signature=signature
)
db = db_session.get_db_read_replica()
with db.scoped_session() as session:
user = (
session.query(User.user_id)
.filter(
# Convert checksum wallet to lowercase
User.wallet == wallet.lower(),
User.is_current == True,
)
# In the case that multiple wallets match (not enforced on the data layer),
# pick the user id that is lowest (created first).
.order_by(User.user_id.asc())
.first()
)
if user:
authed_user_id = user.user_id
logger.info(
f"auth_middleware.py | authed_user_id: {authed_user_id}"
)
return func(*args, **kwargs, authed_user_id=authed_user_id)
def get(self):
args = verify_token_parser.parse_args()
# 1. Break JWT into parts
token_parts = args["token"].split(".")
if not len(token_parts) == 3:
abort_bad_request_param("token", ns)
# 2. Decode the signature
try:
signature = base64.urlsafe_b64decode(token_parts[2] + "==")
except Exception:
ns.abort(400, "The JWT signature could not be decoded.")
signature = signature.decode()
base64_header = token_parts[0]
base64_payload = token_parts[1]
message = f"{base64_header}.{base64_payload}"
# 3. Recover message from signature
web3 = web3_provider.get_web3()
wallet = None
encoded_message = encode_defunct(text=message)
try:
wallet = web3.eth.account.recover_message(
encoded_message,
signature=signature,
)
except Exception:
ns.abort(
404, "The JWT signature is invalid - wallet could not be recovered."
)
if not wallet:
ns.abort(
404, "The JWT signature is invalid - wallet could not be recovered."
)
# 4. Check that user from payload matches the user from the wallet from the signature
try:
stringified_payload = base64.urlsafe_b64decode(base64_payload + "==")
payload = json.loads(stringified_payload)
except Exception:
ns.abort(400, "JWT payload could not be decoded.")
wallet_user_id = get_user_with_wallet(wallet)
if not wallet_user_id or wallet_user_id != payload["userId"]:
ns.abort(
404,
"The JWT signature is invalid - the wallet does not match the user.",
)
# 5. Send back the decoded payload
return success_response(payload)
def get_health(args, use_redis_cache=True):
"""
Gets health status for the service
:param args: dictionary
:param args.verbose: bool
if True, returns db connection information
:param args.healthy_block_diff: int
determines the point at which a block difference is considered unhealthy
:param args.enforce_block_diff: bool
if true and the block difference is unhealthy an error is returned
:rtype: (dictionary, bool)
:return: tuple of health results and a boolean indicating an error
"""
redis = redis_connection.get_redis()
web3 = web3_provider.get_web3()
verbose = args.get("verbose")
enforce_block_diff = args.get("enforce_block_diff")
qs_healthy_block_diff = args.get("healthy_block_diff")
# If healthy block diff is given in url and positive, override config value
healthy_block_diff = qs_healthy_block_diff if qs_healthy_block_diff is not None \
and qs_healthy_block_diff >= 0 else default_healthy_block_diff
latest_block_num = None
latest_block_hash = None
latest_indexed_block_num = None
latest_indexed_block_hash = None
if use_redis_cache:
# get latest blockchain state from redis cache, or fallback to chain if None
latest_block_num, latest_block_hash = get_latest_chain_block_set_if_nx(
redis, web3)
# get latest db state from redis cache
latest_indexed_block_num = redis.get(
most_recent_indexed_block_redis_key)
if latest_indexed_block_num is not None:
latest_indexed_block_num = int(latest_indexed_block_num)
latest_indexed_block_hash = redis.get(
most_recent_indexed_block_hash_redis_key)
if latest_indexed_block_hash is not None:
latest_indexed_block_hash = latest_indexed_block_hash.decode(
"utf-8")
# fetch latest blockchain state from web3 if:
# we explicitly don't want to use redis cache or
# value from redis cache is None
if not use_redis_cache or latest_block_num is None or latest_block_hash is None:
# get latest blockchain state from web3
latest_block = web3.eth.getBlock("latest", True)
latest_block_num = latest_block.number
latest_block_hash = latest_block.hash.hex()
# fetch latest db state if:
# we explicitly don't want to use redis cache or
# value from redis cache is None
if not use_redis_cache or latest_indexed_block_num is None or latest_indexed_block_hash is None:
db_block_state = _get_db_block_state()
latest_indexed_block_num = db_block_state["number"] or 0
latest_indexed_block_hash = db_block_state["blockhash"]
trending_tracks_age_sec = get_elapsed_time_redis(
redis, trending_tracks_last_completion_redis_key)
trending_playlists_age_sec = get_elapsed_time_redis(
redis, trending_playlists_last_completion_redis_key)
# Get system information monitor values
sys_info = monitors.get_monitors([
MONITORS[monitor_names.database_size],
MONITORS[monitor_names.database_connections],
MONITORS[monitor_names.total_memory],
MONITORS[monitor_names.used_memory],
MONITORS[monitor_names.filesystem_size],
MONITORS[monitor_names.filesystem_used],
MONITORS[monitor_names.received_bytes_per_sec],
MONITORS[monitor_names.transferred_bytes_per_sec],
MONITORS[monitor_names.redis_total_memory]
])
health_results = {
"web": {
"blocknumber": latest_block_num,
"blockhash": latest_block_hash,
},
"db": {
"number": latest_indexed_block_num,
"blockhash": latest_indexed_block_hash
},
"git": os.getenv("GIT_SHA"),
"trending_tracks_age_sec": trending_tracks_age_sec,
"trending_playlists_age_sec": trending_playlists_age_sec,
"number_of_cpus": number_of_cpus,
**sys_info
}
block_difference = abs(health_results["web"]["blocknumber"] -
health_results["db"]["number"])
health_results["block_difference"] = block_difference
health_results[
"maximum_healthy_block_difference"] = default_healthy_block_diff
health_results.update(disc_prov_version)
if verbose:
# DB connections check
db_connections_json, error = _get_db_conn_state()
health_results["db_connections"] = db_connections_json
if error:
return health_results, error
# Return error on unhealthy block diff if requested.
if enforce_block_diff and health_results[
"block_difference"] > healthy_block_diff:
return health_results, True
return health_results, False
def get_health(args: GetHealthArgs,
use_redis_cache: bool = True) -> Tuple[Dict, bool]:
"""
Gets health status for the service
Returns a tuple of health results and a boolean indicating an error
"""
redis = redis_connection.get_redis()
web3 = web3_provider.get_web3()
verbose = args.get("verbose")
enforce_block_diff = args.get("enforce_block_diff")
qs_healthy_block_diff = cast(Optional[int], args.get("healthy_block_diff"))
challenge_events_age_max_drift = args.get("challenge_events_age_max_drift")
plays_count_max_drift = args.get("plays_count_max_drift")
# If healthy block diff is given in url and positive, override config value
healthy_block_diff = (
qs_healthy_block_diff if qs_healthy_block_diff is not None
and qs_healthy_block_diff >= 0 else default_healthy_block_diff)
latest_block_num: Optional[int] = None
latest_block_hash: Optional[str] = None
latest_indexed_block_num: Optional[int] = None
latest_indexed_block_hash: Optional[str] = None
if use_redis_cache:
# get latest blockchain state from redis cache, or fallback to chain if None
latest_block_num, latest_block_hash = get_latest_chain_block_set_if_nx(
redis, web3)
# get latest db state from redis cache
latest_indexed_block_num = redis.get(
most_recent_indexed_block_redis_key)
if latest_indexed_block_num is not None:
latest_indexed_block_num = int(latest_indexed_block_num)
latest_indexed_block_hash_bytes = redis.get(
most_recent_indexed_block_hash_redis_key)
if latest_indexed_block_hash_bytes is not None:
latest_indexed_block_hash = latest_indexed_block_hash_bytes.decode(
"utf-8")
else:
# Get latest blockchain state from web3
try:
latest_block = web3.eth.get_block("latest", True)
latest_block_num = latest_block.number
latest_block_hash = latest_block.hash.hex()
except Exception as e:
logger.error(f"Could not get latest block from chain: {e}")
# fetch latest db state if:
# we explicitly don't want to use redis cache or
# value from redis cache is None
if (not use_redis_cache or latest_indexed_block_num is None
or latest_indexed_block_hash is None):
db_block_state = _get_db_block_state()
latest_indexed_block_num = db_block_state["number"] or 0
latest_indexed_block_hash = db_block_state["blockhash"]
play_health_info = get_play_health_info(redis, plays_count_max_drift)
rewards_manager_health_info = get_rewards_manager_health_info(redis)
user_bank_health_info = get_user_bank_health_info(redis)
spl_audio_info = get_spl_audio_info(redis)
reactions_health_info = get_reactions_health_info(
redis,
args.get("reactions_max_indexing_drift"),
args.get("reactions_max_last_reaction_drift"),
)
trending_tracks_age_sec = get_elapsed_time_redis(
redis, trending_tracks_last_completion_redis_key)
trending_playlists_age_sec = get_elapsed_time_redis(
redis, trending_playlists_last_completion_redis_key)
challenge_events_age_sec = get_elapsed_time_redis(
redis, challenges_last_processed_event_redis_key)
user_balances_age_sec = get_elapsed_time_redis(
redis, user_balances_refresh_last_completion_redis_key)
num_users_in_lazy_balance_refresh_queue = int(
redis.scard(LAZY_REFRESH_REDIS_PREFIX))
num_users_in_immediate_balance_refresh_queue = int(
redis.scard(IMMEDIATE_REFRESH_REDIS_PREFIX))
last_scanned_block_for_balance_refresh = redis_get_or_restore(
redis, eth_indexing_last_scanned_block_key)
index_eth_age_sec = get_elapsed_time_redis(
redis, index_eth_last_completion_redis_key)
last_scanned_block_for_balance_refresh = (
int(last_scanned_block_for_balance_refresh)
if last_scanned_block_for_balance_refresh else None)
# Get system information monitor values
sys_info = monitors.get_monitors([
MONITORS[monitor_names.database_size],
MONITORS[monitor_names.database_connections],
MONITORS[monitor_names.total_memory],
MONITORS[monitor_names.used_memory],
MONITORS[monitor_names.filesystem_size],
MONITORS[monitor_names.filesystem_used],
MONITORS[monitor_names.received_bytes_per_sec],
MONITORS[monitor_names.transferred_bytes_per_sec],
MONITORS[monitor_names.redis_total_memory],
])
health_results = {
"web": {
"blocknumber": latest_block_num,
"blockhash": latest_block_hash,
},
"db": {
"number": latest_indexed_block_num,
"blockhash": latest_indexed_block_hash,
},
"git": os.getenv("GIT_SHA"),
"trending_tracks_age_sec": trending_tracks_age_sec,
"trending_playlists_age_sec": trending_playlists_age_sec,
"challenge_last_event_age_sec": challenge_events_age_sec,
"user_balances_age_sec": user_balances_age_sec,
"num_users_in_lazy_balance_refresh_queue":
num_users_in_lazy_balance_refresh_queue,
"num_users_in_immediate_balance_refresh_queue":
num_users_in_immediate_balance_refresh_queue,
"last_scanned_block_for_balance_refresh":
last_scanned_block_for_balance_refresh,
"index_eth_age_sec": index_eth_age_sec,
"number_of_cpus": number_of_cpus,
**sys_info,
"plays": play_health_info,
"rewards_manager": rewards_manager_health_info,
"user_bank": user_bank_health_info,
"openresty_public_key": openresty_public_key,
"spl_audio_info": spl_audio_info,
"reactions": reactions_health_info,
"infra_setup": infra_setup,
}
if latest_block_num is not None and latest_indexed_block_num is not None:
block_difference = abs(latest_block_num - latest_indexed_block_num)
else:
# If we cannot get a reading from chain about what the latest block is,
# we set the difference to be an unhealthy amount
block_difference = default_healthy_block_diff + 1
health_results["block_difference"] = block_difference
health_results[
"maximum_healthy_block_difference"] = default_healthy_block_diff
health_results.update(disc_prov_version)
# Check that this node meets the minimum system requirements
num_cpus: int = cast(int, health_results["number_of_cpus"] or 0)
total_memory: int = cast(int, health_results["total_memory"] or 0)
filesystem_size: int = cast(int, health_results["filesystem_size"] or 0)
if (num_cpus < min_number_of_cpus or total_memory < min_total_memory
or filesystem_size < min_filesystem_size):
health_results["meets_min_requirements"] = False
# TODO - this will become strictly enforced in upcoming service versions and return with error
else:
health_results["meets_min_requirements"] = True
if verbose:
# Elasticsearch health
if esclient:
health_results["elasticsearch"] = get_elasticsearch_health_info(
esclient, latest_indexed_block_num)
# DB connections check
db_connections_json, db_connections_error = _get_db_conn_state()
health_results["db_connections"] = db_connections_json
location = get_location()
health_results.update(location)
if db_connections_error:
return health_results, db_connections_error
query_insights_json, query_insights_error = _get_query_insights()
health_results["query_insights"] = query_insights_json
if query_insights_error:
return health_results, query_insights_error
table_size_info_json = monitors.get_monitors([
MONITORS[monitor_names.table_size_info],
])
health_results["tables"] = table_size_info_json
unhealthy_blocks = bool(enforce_block_diff
and block_difference > healthy_block_diff)
unhealthy_challenges = bool(
challenge_events_age_max_drift and challenge_events_age_sec
and challenge_events_age_sec > challenge_events_age_max_drift)
is_unhealthy = (unhealthy_blocks or unhealthy_challenges
or play_health_info["is_unhealthy"]
or reactions_health_info["is_unhealthy"])
return health_results, is_unhealthy
def get_health(args, use_redis_cache=True):
"""
Gets health status for the service
:param args: dictionary
:param args.verbose: bool
if True, returns db connection information
:param args.healthy_block_diff: int
determines the point at which a block difference is considered unhealthy
:param args.enforce_block_diff: bool
if true and the block difference is unhealthy an error is returned
:rtype: (dictionary, bool)
:return: tuple of health results and a boolean indicating an error
"""
redis = redis_connection.get_redis()
web3 = web3_provider.get_web3()
verbose = args.get("verbose")
enforce_block_diff = args.get("enforce_block_diff")
qs_healthy_block_diff = args.get("healthy_block_diff")
# If healthy block diff is given in url and positive, override config value
healthy_block_diff = qs_healthy_block_diff if qs_healthy_block_diff is not None \
and qs_healthy_block_diff >= 0 else default_healthy_block_diff
latest_block_num = None
latest_block_hash = None
# Get latest web block info
if use_redis_cache:
stored_latest_block_num = redis.get(latest_block_redis_key)
if stored_latest_block_num is not None:
latest_block_num = int(stored_latest_block_num)
stored_latest_blockhash = redis.get(latest_block_hash_redis_key)
if stored_latest_blockhash is not None:
latest_block_hash = stored_latest_blockhash.decode("utf-8")
if latest_block_num is None or latest_block_hash is None:
latest_block = web3.eth.getBlock("latest", True)
latest_block_num = latest_block.number
latest_block_hash = latest_block.hash.hex()
latest_indexed_block_num = None
latest_indexed_block_hash = None
# Get latest indexed block info
if use_redis_cache:
latest_indexed_block_num = redis.get(
most_recent_indexed_block_redis_key)
if latest_indexed_block_num is not None:
latest_indexed_block_num = int(latest_indexed_block_num)
latest_indexed_block_hash = redis.get(
most_recent_indexed_block_hash_redis_key)
if latest_indexed_block_hash is not None:
latest_indexed_block_hash = latest_indexed_block_hash.decode(
"utf-8")
if latest_indexed_block_num is None or latest_indexed_block_hash is None:
db_block_state = _get_db_block_state()
latest_indexed_block_num = db_block_state["number"] or 0
latest_indexed_block_hash = db_block_state["blockhash"]
health_results = {
"web": {
"blocknumber": latest_block_num,
"blockhash": latest_block_hash,
},
"db": {
"number": latest_indexed_block_num,
"blockhash": latest_indexed_block_hash
},
"git": os.getenv("GIT_SHA"),
}
block_difference = abs(
health_results["web"]["blocknumber"] - health_results["db"]["number"]
)
health_results["block_difference"] = block_difference
health_results["maximum_healthy_block_difference"] = default_healthy_block_diff
health_results.update(disc_prov_version)
if verbose:
# DB connections check
db_connections_json, error = _get_db_conn_state()
health_results["db_connections"] = db_connections_json
if error:
return health_results, error
# Return error on unhealthy block diff if requested.
if enforce_block_diff and health_results["block_difference"] > healthy_block_diff:
return health_results, True
return health_results, False
# pylint: disable=no-name-in-module
from eth_account.messages import encode_defunct
from flask import jsonify
from src.queries.get_health import get_latest_chain_block_set_if_nx
from src.queries.get_sol_plays import get_sol_play_health_info
# pylint: disable=R0401
from src.utils import helpers, web3_provider
from src.utils.config import shared_config
from src.utils.redis_constants import most_recent_indexed_block_redis_key
from web3 import Web3
from web3.auto import w3
redis_url = shared_config["redis"]["url"]
redis_conn = redis.Redis.from_url(url=redis_url)
web3_connection = web3_provider.get_web3()
logger = logging.getLogger(__name__)
disc_prov_version = helpers.get_discovery_provider_version()
# Subclass JSONEncoder
class DateTimeEncoder(json.JSONEncoder):
# Override the default method
def default(self, o): # pylint: disable=E0202
if isinstance(o, (datetime.date, datetime.datetime)):
# the Z is required in JS date format
return o.isoformat() + " Z"
return json.JSONEncoder.default(self, o)
def error_response(error, error_code=500):