def test_get_cached_app_metrics(redis_mock):
metrics = {
old_time.strftime(datetime_format_secondary): {
'some-app': 1,
'other-app': 2
},
recent_time_1.strftime(datetime_format_secondary): {
'another-app': 1,
'some-other-app': 2
},
recent_time_2.strftime(datetime_format_secondary): {
'top-app': 1,
'some-app': 2,
'another-app': 3
}
}
redis_set_and_dump(redis_mock, personal_app_metrics, json.dumps(metrics))
result = get_redis_metrics(redis_mock, start_time_obj,
personal_app_metrics)
assert len(result.items()) == 4
assert result['another-app'] == 4
assert result['some-other-app'] == 2
assert result['top-app'] == 1
assert result['some-app'] == 2
def test_get_cached_app_metrics(redis_mock):
metrics = {
old_time.strftime(datetime_format_secondary): {
"some-app": 1,
"other-app": 2
},
recent_time_1.strftime(datetime_format_secondary): {
"another-app": 1,
"some-other-app": 2,
},
recent_time_2.strftime(datetime_format_secondary): {
"top-app": 1,
"some-app": 2,
"another-app": 3,
},
}
redis_set_and_dump(redis_mock, personal_app_metrics, json.dumps(metrics))
result = get_redis_metrics(redis_mock, start_time_obj,
personal_app_metrics)
assert len(result.items()) == 4
assert result["another-app"] == 4
assert result["some-other-app"] == 2
assert result["top-app"] == 1
assert result["some-app"] == 2
def update_summed_unique_metrics(now, ip):
thirty_one_days_ago = now - timedelta(days=31)
thirty_one_days_ago_str = thirty_one_days_ago.strftime(day_format)
yesterday = now - timedelta(days=1)
yesterday_str = yesterday.strftime(day_format)
today_str = now.strftime(day_format)
this_month_str = f"{today_str[:7]}/01"
summed_unique_daily_metrics_str = redis_get_or_restore(REDIS, summed_unique_daily_metrics)
summed_unique_daily_metrics_obj = json.loads(summed_unique_daily_metrics_str) \
if summed_unique_daily_metrics_str else {}
if today_str not in summed_unique_daily_metrics_obj:
summed_unique_daily_metrics_obj[today_str] = [ip]
elif ip not in summed_unique_daily_metrics_obj[today_str]:
summed_unique_daily_metrics_obj[today_str] = summed_unique_daily_metrics_obj[today_str] + [ip]
summed_unique_daily_metrics_obj = {timestamp: ips for timestamp, ips in summed_unique_daily_metrics_obj.items() \
if timestamp >= yesterday_str}
redis_set_and_dump(REDIS, summed_unique_daily_metrics, json.dumps(summed_unique_daily_metrics_obj))
summed_unique_monthly_metrics_str = redis_get_or_restore(REDIS, summed_unique_monthly_metrics)
summed_unique_monthly_metrics_obj = json.loads(summed_unique_monthly_metrics_str) \
if summed_unique_monthly_metrics_str else {}
if this_month_str not in summed_unique_monthly_metrics_obj:
summed_unique_monthly_metrics_obj[this_month_str] = [ip]
elif ip not in summed_unique_monthly_metrics_obj[this_month_str]:
summed_unique_monthly_metrics_obj[this_month_str] = summed_unique_monthly_metrics_obj[this_month_str] + [ip]
summed_unique_monthly_metrics_obj = {timestamp: ips for timestamp, ips \
in summed_unique_monthly_metrics_obj.items() if timestamp >= thirty_one_days_ago_str}
redis_set_and_dump(REDIS, summed_unique_monthly_metrics, json.dumps(summed_unique_monthly_metrics_obj))
def update_personal_metrics(key, old_timestamp, timestamp, value, metric_type):
values_str = redis_get_or_restore(REDIS, key)
values = json.loads(values_str) if values_str else {}
if timestamp in values:
values[timestamp][value] = values[timestamp][value] + 1 if value in values[timestamp] else 1
else:
values[timestamp] = {value: 1}
# clean up and update cached metrics
updated_metrics = {timestamp: metrics for timestamp, metrics in values.items() if timestamp > old_timestamp}
if updated_metrics:
redis_set_and_dump(REDIS, key, json.dumps(updated_metrics))
logger.info(f"updated cached personal {metric_type} metrics")
def consolidate_metrics_from_other_nodes(self, db, redis):
"""
Get recent route and app metrics from all other discovery nodes
and merge with this node's metrics so that this node will be aware
of all the metrics across users hitting different providers
"""
all_other_nodes = get_all_other_nodes()
visited_node_timestamps_str = redis_get_or_restore(redis, metrics_visited_nodes)
visited_node_timestamps = json.loads(visited_node_timestamps_str) if visited_node_timestamps_str else {}
now = datetime.utcnow()
one_iteration_ago = now - timedelta(minutes=METRICS_INTERVAL)
one_iteration_ago_str = one_iteration_ago.strftime(datetime_format_secondary)
end_time = now.strftime(datetime_format_secondary)
# personal unique metrics for the day and the month
summed_unique_metrics = get_summed_unique_metrics(now)
summed_unique_daily_count = summed_unique_metrics['daily']
summed_unique_monthly_count = summed_unique_metrics['monthly']
# Merge & persist metrics for our personal node
personal_route_metrics_str = redis_get_or_restore(redis, personal_route_metrics)
personal_route_metrics_dict = json.loads(personal_route_metrics_str) if personal_route_metrics_str else {}
new_personal_route_metrics = {}
for timestamp, metrics in personal_route_metrics_dict.items():
if timestamp > one_iteration_ago_str:
for ip, count in metrics.items():
if ip in new_personal_route_metrics:
new_personal_route_metrics[ip] += count
else:
new_personal_route_metrics[ip] = count
personal_app_metrics_str = redis_get_or_restore(redis, personal_app_metrics)
personal_app_metrics_dict = json.loads(personal_app_metrics_str) if personal_app_metrics_str else {}
new_personal_app_metrics = {}
for timestamp, metrics in personal_app_metrics_dict.items():
if timestamp > one_iteration_ago_str:
for app_name, count in metrics.items():
if app_name in new_personal_app_metrics:
new_personal_app_metrics[app_name] += count
else:
new_personal_app_metrics[app_name] = count
merge_route_metrics(new_personal_route_metrics, end_time, db)
merge_app_metrics(new_personal_app_metrics, end_time, db)
# Merge & persist metrics for other nodes
for node in all_other_nodes:
start_time_str = visited_node_timestamps[node] if node in visited_node_timestamps else one_iteration_ago_str
start_time_obj = datetime.strptime(start_time_str, datetime_format_secondary)
start_time = int(start_time_obj.timestamp())
new_route_metrics, new_app_metrics = get_metrics(node, start_time)
logger.info(f"did attempt to receive route and app metrics from {node} at {start_time_obj} ({start_time})")
# add other nodes' summed unique daily and monthly counts to this node's
if new_route_metrics:
logger.info(f"summed unique metrics from {node}: {new_route_metrics['summed']}")
summed_unique_daily_count += new_route_metrics['summed']['daily']
summed_unique_monthly_count += new_route_metrics['summed']['monthly']
new_route_metrics = new_route_metrics['deduped']
merge_route_metrics(new_route_metrics or {}, end_time, db)
merge_app_metrics(new_app_metrics or {}, end_time, db)
if new_route_metrics is not None and new_app_metrics is not None:
visited_node_timestamps[node] = end_time
# persist updated summed unique counts
persist_summed_unique_counts(db, end_time, summed_unique_daily_count, summed_unique_monthly_count)
logger.info(f"visited node timestamps: {visited_node_timestamps}")
if visited_node_timestamps:
redis_set_and_dump(redis, metrics_visited_nodes, json.dumps(visited_node_timestamps))
def merge_metrics(metrics, end_time, metric_type, db):
"""
Merge this node's metrics to those received from other discovery nodes:
Update unique and total, daily and monthly metrics for routes and apps
Dump the cached metrics so that if this node temporarily goes down,
we can recover the IPs and app names to perform the calculation and deduplication
when the node comes back up
Clean up old metrics from cache
Persist metrics in the database
"""
logger.info(
f"about to merge {metric_type} metrics: {len(metrics)} new entries")
day = end_time.split(':')[0]
month = f"{day[:7]}/01"
daily_key = daily_route_metrics if metric_type == 'route' else daily_app_metrics
daily_metrics_str = redis_get_or_restore(REDIS, daily_key)
daily_metrics = json.loads(daily_metrics_str) if daily_metrics_str else {}
monthly_key = monthly_route_metrics if metric_type == 'route' else monthly_app_metrics
monthly_metrics_str = redis_get_or_restore(REDIS, monthly_key)
monthly_metrics = json.loads(
monthly_metrics_str) if monthly_metrics_str else {}
if day not in daily_metrics:
daily_metrics[day] = {}
if month not in monthly_metrics:
monthly_metrics[month] = {}
# only relevant for unique users metrics
unique_daily_count = 0
unique_monthly_count = 0
# only relevant for app metrics
app_count = {}
# update daily and monthly metrics, which could be route metrics or app metrics
# if route metrics, new_value and new_count would be an IP and the number of requests from it
# otherwise, new_value and new_count would be an app and the number of requests from it
for new_value, new_count in metrics.items():
if metric_type == 'route' and new_value not in daily_metrics[day]:
unique_daily_count += 1
if metric_type == 'route' and new_value not in monthly_metrics[month]:
unique_monthly_count += 1
if metric_type == 'app':
app_count[new_value] = new_count
daily_metrics[day][new_value] = daily_metrics[day][new_value] + new_count \
if new_value in daily_metrics[day] else new_count
monthly_metrics[month][new_value] = monthly_metrics[month][new_value] + new_count \
if new_value in monthly_metrics[month] else new_count
# clean up metrics METRICS_INTERVAL after the end of the day from daily_metrics
yesterday_str = (datetime.utcnow() -
timedelta(days=1)).strftime(datetime_format_secondary)
daily_metrics = {timestamp: metrics for timestamp, metrics in daily_metrics.items() \
if timestamp > yesterday_str}
if daily_metrics:
redis_set_and_dump(REDIS, daily_key, json.dumps(daily_metrics))
logger.info(f"updated cached daily {metric_type} metrics")
# clean up metrics METRICS_INTERVAL after the end of the month from monthly_metrics
thirty_one_days_ago = (
datetime.utcnow() -
timedelta(days=31)).strftime(datetime_format_secondary)
monthly_metrics = {timestamp: metrics for timestamp, metrics in monthly_metrics.items() \
if timestamp > thirty_one_days_ago}
if monthly_metrics:
redis_set_and_dump(REDIS, monthly_key, json.dumps(monthly_metrics))
logger.info(f"updated cached monthly {metric_type} metrics")
# persist aggregated metrics from other nodes
day_obj = datetime.strptime(day, day_format).date()
month_obj = datetime.strptime(month, day_format).date()
if metric_type == 'route':
persist_route_metrics(db, day_obj, month_obj, sum(metrics.values()),
unique_daily_count, unique_monthly_count)
else:
persist_app_metrics(db, day_obj, month_obj, app_count)
def get_play_health_info(
redis: Redis, plays_count_max_drift: Optional[int]) -> PlayHealthInfo:
if redis is None:
raise Exception("Invalid arguments for get_play_health_info")
current_time_utc = datetime.utcnow()
# Fetch plays info from Solana
sol_play_info = get_sol_play_health_info(redis, current_time_utc)
# If play count max drift provided, perform comparison
is_unhealthy_sol_plays = bool(
plays_count_max_drift
and plays_count_max_drift < sol_play_info["time_diff"])
# If unhealthy sol plays, this will be overwritten
time_diff_general = sol_play_info["time_diff"]
if is_unhealthy_sol_plays or not plays_count_max_drift:
# Calculate time diff from now to latest play
latest_db_play = redis_get_or_restore(redis, latest_legacy_play_db_key)
if not latest_db_play:
# Query and cache latest db play if found
latest_db_play = get_latest_play()
if latest_db_play:
redis_set_and_dump(redis, latest_legacy_play_db_key,
latest_db_play.timestamp())
else:
# Decode bytes into float for latest timestamp
latest_db_play = float(latest_db_play.decode())
latest_db_play = datetime.utcfromtimestamp(latest_db_play)
oldest_unarchived_play = redis_get_or_restore(
redis, oldest_unarchived_play_key)
if not oldest_unarchived_play:
# Query and cache oldest unarchived play
oldest_unarchived_play = get_oldest_unarchived_play()
if oldest_unarchived_play:
redis_set_and_dump(
redis,
oldest_unarchived_play_key,
oldest_unarchived_play.timestamp(),
)
else:
# Decode bytes into float for latest timestamp
oldest_unarchived_play = float(oldest_unarchived_play.decode())
oldest_unarchived_play = datetime.utcfromtimestamp(
oldest_unarchived_play)
time_diff_general = ((current_time_utc -
latest_db_play).total_seconds()
if latest_db_play else time_diff_general)
is_unhealthy_plays = bool(plays_count_max_drift and
(is_unhealthy_sol_plays and
(plays_count_max_drift < time_diff_general)))
return {
"is_unhealthy": is_unhealthy_plays,
"tx_info": sol_play_info,
"time_diff_general": time_diff_general,
"oldest_unarchived_play_created_at": oldest_unarchived_play,
}