def _get_underground_trending_with_session(
session: Session,
args: GetUndergroundTrendingTrackcArgs,
strategy,
use_request_context=True,
):
current_user_id = args.get("current_user_id", None)
limit, offset = args.get("limit"), args.get("offset")
key = make_underground_trending_cache_key(strategy.version)
(tracks, track_ids) = use_redis_cache(
key, None, make_get_unpopulated_tracks(session, redis_conn, strategy)
)
# Apply limit + offset early to reduce the amount of
# population work we have to do
if limit is not None and offset is not None:
track_ids = track_ids[offset : limit + offset]
tracks = populate_track_metadata(session, track_ids, tracks, current_user_id)
tracks_map = {track["track_id"]: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id, use_request_context)
for track in sorted_tracks:
user = users[track["owner_id"]]
if user:
track["user"] = user
sorted_tracks = list(map(extend_track, sorted_tracks))
return sorted_tracks
def get_trending_tracks(args, strategy):
"""Gets trending by getting the currently cached tracks and then populating them."""
db = get_db_read_replica()
with db.scoped_session() as session:
current_user_id, genre, time = args.get("current_user_id"), args.get(
"genre"), args.get("time", "week")
time_range = "week" if time not in ["week", "month", "year"] else time
key = make_trending_cache_key(time_range, genre, strategy.version)
# Will try to hit cached trending from task, falling back
# to generating it here if necessary and storing it with no TTL
(tracks, track_ids) = use_redis_cache(
key, None,
make_generate_unpopulated_trending(session, genre, time_range,
strategy))
# populate track metadata
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return sorted_tracks
def _get_underground_trending(args, strategy):
db = get_db_read_replica()
with db.scoped_session() as session:
current_user_id = args.get("current_user_id", None)
limit, offset = args.get("limit"), args.get("offset")
key = make_underground_trending_cache_key(strategy.version)
(tracks, track_ids) = use_redis_cache(
key, None, make_get_unpopulated_tracks(session, redis, strategy))
# Apply limit + offset early to reduce the amount of
# population work we have to do
if limit is not None and offset is not None:
track_ids = track_ids[offset:limit + offset]
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
sorted_tracks = list(map(extend_track, sorted_tracks))
return sorted_tracks
def get_full_trending_playlists(request, args, strategy):
offset, limit = format_offset(args), format_limit(args, TRENDING_LIMIT)
current_user_id, time = args.get("user_id"), args.get("time", "week")
time = "week" if time not in ["week", "month", "year"] else time
# If we have a user_id, we call into `get_trending_playlist`
# which fetches the cached unpopulated tracks and then
# populates metadata. Otherwise, just
# retrieve the last cached value.
#
# If current_user_id,
# apply limit + offset inside the cached calculation.
# Otherwise, apply it here.
if current_user_id:
args = {
'time': time,
'with_tracks': True,
'limit': limit,
'offset': offset
}
decoded = decode_string_id(current_user_id)
args["current_user_id"] = decoded
playlists = get_trending_playlists(args, strategy)
else:
args = {
'time': time,
'with_tracks': True,
}
key = get_trending_cache_key(to_dict(request.args), request.path)
playlists = use_redis_cache(
key, TRENDING_TTL_SEC,
lambda: get_trending_playlists(args, strategy))
playlists = playlists[offset:limit + offset]
return playlists
def get_time_trending(cache_args, time, limit):
time_params = {**cache_args, 'time':time}
time_cache_key = extract_key(request_cache_path, time_params.items())
time_trending = use_redis_cache(time_cache_key, TRENDING_TTL_SEC, lambda: get_trending(time_params))
time_trending_track_ids = [{"track_id": track['track_id']} for track in time_trending]
time_trending_track_ids = time_trending_track_ids[:limit]
return time_trending_track_ids
def get_full_recommended_tracks(request, args, strategy):
# Attempt to use the cached tracks list
if args["user_id"] is not None:
full_recommended = get_recommended_tracks(args, strategy)
else:
key = get_trending_cache_key(to_dict(request.args), request.path)
full_recommended = use_redis_cache(
key, TRENDING_TTL_SEC,
lambda: get_recommended_tracks(args, strategy))
return full_recommended
def get_trending_tracks(args):
limit, offset, current_user_id = args.get("limit"), args.get(
"offset"), args.get("current_user_id")
db = get_db_read_replica()
time = args.get('time')
query_time = None if time not in ["day", "week", "month", "year"] else time
with db.scoped_session() as session:
def get_unpopulated_trending():
trending_tracks = generate_trending(session, query_time,
args.get('genre', None), limit,
offset)
track_scores = [
z(time, track) for track in trending_tracks['listen_counts']
]
sorted_track_scores = sorted(track_scores,
key=lambda k: k['score'],
reverse=True)
track_ids = [track['track_id'] for track in sorted_track_scores]
tracks = get_unpopulated_tracks(session, track_ids)
return (tracks, track_ids)
# get scored trending tracks, either
# through cached redis value, or through `get_unpopulated_trending`
cache_keys = {"genre": args.get("genre"), "time": args.get("time")}
key = extract_key(f"generated-trending:{request.path}",
cache_keys.items())
(tracks, track_ids) = use_redis_cache(key, SCORES_CACHE_DURATION_SEC,
get_unpopulated_trending)
# populate track metadata
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return sorted_tracks
def get_full_trending(request, args, strategy):
offset = format_offset(args)
limit = format_limit(args, TRENDING_LIMIT)
key = get_trending_cache_key(to_dict(request.args), request.path)
# Attempt to use the cached tracks list
if args['user_id'] is not None:
full_trending = get_trending(args, strategy)
else:
full_trending = use_redis_cache(key, TRENDING_TTL_SEC,
lambda: get_trending(args, strategy))
trending_tracks = full_trending[offset:limit + offset]
return trending_tracks
def get(self):
args = full_random_track_parser.parse_args()
limit = format_limit(args, default_limit=DEFAULT_RANDOM_LIMIT)
args['limit'] = max(TRENDING_LIMIT, limit)
key = self.get_cache_key()
# Attempt to use the cached tracks list
if args['user_id'] is not None:
full_random = get_random_tracks(args)
else:
full_random = use_redis_cache(key, TRENDING_TTL_SEC,
lambda: get_random_tracks(args))
random = full_random[:limit]
return success_response(random)
def get(self):
args = full_trending_parser.parse_args()
offset = format_offset(args)
limit = format_limit(args, TRENDING_LIMIT)
key = self.get_cache_key()
# Attempt to use the cached tracks list
if args['user_id'] is not None:
full_trending = get_trending(args)
else:
full_trending = use_redis_cache(key, TRENDING_TTL_SEC,
lambda: get_trending(args))
trending = full_trending[offset:limit + offset]
return success_response(trending)
def get_remix_track_parents(args):
"""Fetch remix parents for a given track.
Args:
args:dict
args.track_id: track id
args.limit: limit
args.offset: offset
args.with_users: with users
args.current_user_id: current user ID
"""
track_id = args.get("track_id")
current_user_id = args.get("current_user_id")
limit = args.get("limit")
offset = args.get("offset")
db = get_db_read_replica()
with db.scoped_session() as session:
def get_unpopulated_remix_parents():
base_query = (
session.query(Track)
.join(
Remix,
and_(
Remix.parent_track_id == Track.track_id,
Remix.child_track_id == track_id,
),
)
.filter(Track.is_current == True, Track.is_unlisted == False)
.order_by(desc(Track.created_at), desc(Track.track_id))
)
tracks = add_query_pagination(base_query, limit, offset).all()
tracks = helpers.query_result_to_list(tracks)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
key = make_cache_key(args)
(tracks, track_ids) = use_redis_cache(
key,
UNPOPULATED_REMIX_PARENTS_CACHE_DURATION_SEC,
get_unpopulated_remix_parents,
)
tracks = populate_track_metadata(session, track_ids, tracks, current_user_id)
if args.get("with_users", False):
add_users_to_tracks(session, tracks, current_user_id)
return tracks
def get_time_trending(cache_args, time, limit, strategy):
time_params = {**cache_args, "time": time}
path = request_cache_path
if strategy.version != DEFAULT_TRENDING_VERSIONS[TrendingType.TRACKS]:
path += f"/{strategy.version.value}"
time_cache_key = extract_key(path, time_params.items())
time_trending = use_redis_cache(
time_cache_key, TRENDING_TTL_SEC,
lambda: get_trending(time_params, strategy))
time_trending_track_ids = [{
"track_id": track["track_id"]
} for track in time_trending]
time_trending_track_ids = time_trending_track_ids[:limit]
return time_trending_track_ids
def get_underground_trending(request, args, strategy):
offset, limit = format_offset(args), format_limit(args, TRENDING_LIMIT)
current_user_id = args.get("user_id")
args = {'limit': limit, 'offset': offset}
# If user ID, let _get_underground_trending
# handle caching + limit + offset
if current_user_id:
decoded = decode_string_id(current_user_id)
args["current_user_id"] = decoded
trending = _get_underground_trending(args, strategy)
else:
# If no user ID, fetch all cached tracks
# and perform pagination here, passing
# no args so we get the full list of tracks.
key = get_trending_cache_key(to_dict(request.args), request.path)
trending = use_redis_cache(
key, TRENDING_TTL_SEC,
lambda: _get_underground_trending({}, strategy))
trending = trending[offset:limit + offset]
return trending
def _get_trending_tracks_with_session(session: Session,
args: GetTrendingTracksArgs,
strategy: BaseTrendingStrategy):
current_user_id, genre, time = (
args.get("current_user_id"),
args.get("genre"),
args.get("time", "week"),
)
time_range = "week" if time not in ["week", "month", "year", "allTime"
] else time
key = make_trending_cache_key(time_range, genre, strategy.version)
# Will try to hit cached trending from task, falling back
# to generating it here if necessary and storing it with no TTL
(tracks, track_ids) = use_redis_cache(
key,
None,
make_generate_unpopulated_trending(session, genre, time_range,
strategy),
)
# populate track metadata
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track["track_id"]: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in sorted_tracks:
user = users[track["owner_id"]]
if user:
track["user"] = user
return sorted_tracks
def get_remixes_of(args):
track_id = args.get("track_id")
current_user_id = args.get("current_user_id")
limit, offset = args.get("limit"), args.get("offset")
db = get_db_read_replica()
with db.scoped_session() as session:
def get_unpopulated_remixes():
# Fetch the parent track to get the track's owner id
parent_track_res = get_unpopulated_tracks(session, [track_id],
False, False)
if not parent_track_res or parent_track_res[0] is None:
raise exceptions.ArgumentError("Invalid track_id provided")
parent_track = parent_track_res[0]
track_owner_id = parent_track['owner_id']
# Create subquery for save counts for sorting
save_count_subquery = create_save_count_subquery(
session, SaveType.track)
# Create subquery for repost counts for sorting
repost_count_subquery = create_repost_count_subquery(
session, RepostType.track)
# Get the 'children' remix tracks
# Use the track owner id to fetch reposted/saved tracks returned first
base_query = (
session.query(
Track
)
.join(
Remix,
and_(
Remix.child_track_id == Track.track_id,
Remix.parent_track_id == track_id
)
).outerjoin(
Save,
and_(
Save.save_item_id == Track.track_id,
Save.save_type == SaveType.track,
Save.is_current == True,
Save.is_delete == False,
Save.user_id == track_owner_id
)
).outerjoin(
Repost,
and_(
Repost.repost_item_id == Track.track_id,
Repost.user_id == track_owner_id,
Repost.repost_type == RepostType.track,
Repost.is_current == True,
Repost.is_delete == False
)
).outerjoin(
repost_count_subquery,
repost_count_subquery.c['id'] == Track.track_id
).outerjoin(
save_count_subquery,
save_count_subquery.c['id'] == Track.track_id
)
.filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False
)
# 1. Co-signed tracks ordered by save + repost count
# 2. Other tracks ordered by save + repost count
.order_by(
desc(
# If there is no "co-sign" for the track (no repost or save from the parent owner),
# defer to secondary sort
case(
[
(and_(Repost.created_at == None,
Save.created_at == None), 0),
],
else_=(
func.coalesce(repost_count_subquery.c.repost_count, 0) + \
func.coalesce(save_count_subquery.c.save_count, 0)
)
)
),
# Order by saves + reposts
desc(
func.coalesce(repost_count_subquery.c.repost_count, 0) + \
func.coalesce(save_count_subquery.c.save_count, 0)
),
# Ties, pick latest track id
desc(Track.track_id)
)
)
(tracks, count) = add_query_pagination(base_query, limit, offset,
True, True)
tracks = tracks.all()
tracks = helpers.query_result_to_list(tracks)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids, count)
key = make_cache_key(args)
(tracks, track_ids,
count) = use_redis_cache(key, UNPOPULATED_REMIXES_CACHE_DURATION_SEC,
get_unpopulated_remixes)
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
if args.get("with_users", False):
add_users_to_tracks(session, tracks, current_user_id)
return {'tracks': tracks, 'count': count}
def get_tracks_including_unlisted(args):
"""Fetch a track, allowing unlisted.
Args:
args: dict
args.identifiers: array of { handle, id, url_title} dicts
args.current_user_id: optional current user ID
args.filter_deleted: filter deleted tracks
args.with_users: include users in unlisted tracks
"""
tracks = []
identifiers = args["identifiers"]
for i in identifiers:
helpers.validate_arguments(i, ["handle", "id", "url_title"])
current_user_id = args.get("current_user_id")
db = get_db_read_replica()
with db.scoped_session() as session:
def get_unpopulated_track():
base_query = session.query(Track)
filter_cond = []
# Create filter conditions as a list of `and` clauses
for i in identifiers:
filter_cond.append(
and_(Track.is_current == True, Track.track_id == i["id"]))
# Pass array of `and` clauses into an `or` clause as destructured *args
base_query = base_query.filter(or_(*filter_cond))
# Allow filtering of deletes
# Note: There is no standard for boolean url parameters, and any value (including 'false')
# will be evaluated as true, so an explicit check is made for true
if "filter_deleted" in args:
filter_deleted = args.get("filter_deleted")
if filter_deleted:
base_query = base_query.filter(Track.is_delete == False)
# Perform the query
# TODO: pagination is broken with unlisted tracks
query_results = paginate_query(base_query).all()
tracks = helpers.query_result_to_list(query_results)
# Mapping of track_id -> track object from request;
# used to check route_id when iterating through identifiers
identifiers_map = {track["id"]: track for track in identifiers}
# If the track is unlisted and the generated route_id does not match the route_id in db,
# filter track out from response
def filter_fn(track):
input_track = identifiers_map[track["track_id"]]
route_id = helpers.create_track_route_id(
input_track["url_title"], input_track["handle"])
return not track["is_unlisted"] or track["route_id"] == route_id
tracks = list(filter(filter_fn, tracks))
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
key = make_cache_key(args)
(tracks,
track_ids) = use_redis_cache(key,
UNPOPULATED_TRACK_CACHE_DURATION_SEC,
get_unpopulated_track)
# Add users
if args.get("with_users", False):
user_id_list = get_users_ids(tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in tracks:
user = users[track["owner_id"]]
if user:
track["user"] = user
# Populate metadata
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
return tracks
def get_trending_playlists(args, strategy):
"""Returns Trending Playlists. Checks Redis cache for unpopulated playlists."""
db = get_db_read_replica()
with db.scoped_session() as session:
current_user_id = args.get("current_user_id", None)
with_tracks = args.get("with_tracks", False)
time = args.get("time")
limit, offset = args.get("limit"), args.get("offset")
key = make_trending_cache_key(time, strategy.version)
# Get unpopulated playlists,
# cached if it exists.
(playlists, playlist_ids) = use_redis_cache(
key, None, make_get_unpopulated_playlists(session, time, strategy))
# Apply limit + offset early to reduce the amount of
# population work we have to do
if limit is not None and offset is not None:
playlists = playlists[offset:limit + offset]
playlist_ids = playlist_ids[offset:limit + offset]
# Populate playlist metadata
playlists = populate_playlist_metadata(
session, playlist_ids, playlists,
[RepostType.playlist, RepostType.album],
[SaveType.playlist, SaveType.album], current_user_id)
trimmed_track_ids = None
for playlist in playlists:
playlist["track_count"] = len(playlist["tracks"])
playlist["tracks"] = playlist["tracks"][:PLAYLIST_TRACKS_LIMIT]
# Trim track_ids, which ultimately become added_timestamps
# and need to match the tracks.
trimmed_track_ids = {
track["track_id"]
for track in playlist["tracks"]
}
playlist_track_ids = playlist["playlist_contents"]["track_ids"]
playlist_track_ids = list(
filter(lambda track_id: track_id["track"] in trimmed_track_ids,
playlist_track_ids))
playlist["playlist_contents"]["track_ids"] = playlist_track_ids
playlists_map = {
playlist['playlist_id']: playlist
for playlist in playlists
}
if with_tracks:
# populate track metadata
tracks = []
for playlist in playlists:
playlist_tracks = playlist["tracks"]
tracks.extend(playlist_tracks)
track_ids = [track["track_id"] for track in tracks]
populated_tracks = populate_track_metadata(session, track_ids,
tracks, current_user_id)
# Add users if necessary
add_users_to_tracks(session, populated_tracks, current_user_id)
# Re-associate tracks with playlists
# track_id -> populated_track
populated_track_map = {
track["track_id"]: track
for track in populated_tracks
}
for playlist in playlists_map.values():
for i in range(len(playlist["tracks"])):
track_id = playlist["tracks"][i]["track_id"]
populated = populated_track_map[track_id]
playlist["tracks"][i] = populated
playlist["tracks"] = list(map(extend_track,
playlist["tracks"]))
# re-sort playlists to original order, because populate_playlist_metadata
# unsorts.
sorted_playlists = [
playlists_map[playlist_id] for playlist_id in playlist_ids
]
# Add users to playlists
user_id_list = get_users_ids(sorted_playlists)
users = get_users_by_id(session, user_id_list, current_user_id)
for playlist in sorted_playlists:
user = users[playlist['playlist_owner_id']]
if user:
playlist['user'] = user
# Extend the playlists
playlists = list(map(extend_playlist, playlists))
return sorted_playlists
def get_playlists(args):
playlists = []
current_user_id = args.get("current_user_id")
db = get_db_read_replica()
with db.scoped_session() as session:
def get_unpopulated_playlists():
playlist_query = (session.query(Playlist).filter(
Playlist.is_current == True))
# playlist ids filter if the optional query param is passed in
if "playlist_id" in args:
playlist_id_list = args.get("playlist_id")
try:
playlist_query = playlist_query.filter(
Playlist.playlist_id.in_(playlist_id_list))
except ValueError as e:
raise exceptions.ArgumentError(
"Invalid value found in playlist id list", e)
if "user_id" in args:
user_id = args.get("user_id")
# user id filter if the optional query param is passed in
playlist_query = playlist_query.filter(
Playlist.playlist_owner_id == user_id)
# If no current_user_id, never show hidden playlists
if not current_user_id:
playlist_query = playlist_query.filter(
Playlist.is_private == False)
# Filter out deletes unless we're fetching explicitly by id
if "playlist_id" not in args:
playlist_query = playlist_query.filter(
Playlist.is_delete == False)
playlist_query = playlist_query.order_by(desc(Playlist.created_at))
playlists = paginate_query(playlist_query).all()
playlists = helpers.query_result_to_list(playlists)
# if we passed in a current_user_id, filter out all privte playlists where
# the owner_id doesn't match the current_user_id
if current_user_id:
playlists = list(
filter(
lambda playlist: (not playlist["is_private"]) or
playlist["playlist_owner_id"] == current_user_id,
playlists))
# retrieve playlist ids list
playlist_ids = list(
map(lambda playlist: playlist["playlist_id"], playlists))
return (playlists, playlist_ids)
try:
# Get unpopulated playlists, either via
# redis cache or via get_unpopulated_playlists
key = make_cache_key(args)
(playlists, playlist_ids) = use_redis_cache(
key, UNPOPULATED_PLAYLIST_CACHE_DURATION_SEC,
get_unpopulated_playlists)
# bundle peripheral info into playlist results
playlists = populate_playlist_metadata(
session, playlist_ids, playlists,
[RepostType.playlist, RepostType.album],
[SaveType.playlist, SaveType.album], current_user_id)
if args.get("with_users", False):
user_id_list = get_users_ids(playlists)
users = get_users_by_id(session, user_id_list, current_user_id)
for playlist in playlists:
user = users[playlist['playlist_owner_id']]
if user:
playlist['user'] = user
except sqlalchemy.orm.exc.NoResultFound:
pass
return playlists
def refresh_cnodes_from_identity(self):
registered_cnodes = use_redis_cache(
'registered_cnodes_from_identity', 30,
lambda: fetch_cnode_endpoints_from_identity(self))
return registered_cnodes
