def _get_underground_trending(args, strategy):
db = get_db_read_replica()
with db.scoped_session() as session:
current_user_id = args.get("current_user_id", None)
limit, offset = args.get("limit"), args.get("offset")
key = make_underground_trending_cache_key(strategy.version)
(tracks, track_ids) = use_redis_cache(
key, None, make_get_unpopulated_tracks(session, redis, strategy))
# Apply limit + offset early to reduce the amount of
# population work we have to do
if limit is not None and offset is not None:
track_ids = track_ids[offset:limit + offset]
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
sorted_tracks = list(map(extend_track, sorted_tracks))
return sorted_tracks
def _get_underground_trending_with_session(
session: Session,
args: GetUndergroundTrendingTrackcArgs,
strategy,
use_request_context=True,
):
current_user_id = args.get("current_user_id", None)
limit, offset = args.get("limit"), args.get("offset")
key = make_underground_trending_cache_key(strategy.version)
(tracks, track_ids) = use_redis_cache(
key, None, make_get_unpopulated_tracks(session, redis_conn, strategy)
)
# Apply limit + offset early to reduce the amount of
# population work we have to do
if limit is not None and offset is not None:
track_ids = track_ids[offset : limit + offset]
tracks = populate_track_metadata(session, track_ids, tracks, current_user_id)
tracks_map = {track["track_id"]: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id, use_request_context)
for track in sorted_tracks:
user = users[track["owner_id"]]
if user:
track["user"] = user
sorted_tracks = list(map(extend_track, sorted_tracks))
return sorted_tracks
def get_trending_tracks(args, strategy):
"""Gets trending by getting the currently cached tracks and then populating them."""
db = get_db_read_replica()
with db.scoped_session() as session:
current_user_id, genre, time = args.get("current_user_id"), args.get(
"genre"), args.get("time", "week")
time_range = "week" if time not in ["week", "month", "year"] else time
key = make_trending_cache_key(time_range, genre, strategy.version)
# Will try to hit cached trending from task, falling back
# to generating it here if necessary and storing it with no TTL
(tracks, track_ids) = use_redis_cache(
key, None,
make_generate_unpopulated_trending(session, genre, time_range,
strategy))
# populate track metadata
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return sorted_tracks
def get_top_followee_windowed(type, window, args):
if type != "track":
raise exceptions.ArgumentError(
"Invalid type provided, must be one of 'track'")
valid_windows = ["week", "month", "year"]
if not window or window not in valid_windows:
raise exceptions.ArgumentError(
f"Invalid window provided, must be one of {valid_windows}")
limit = args.get("limit", 25)
current_user_id = args.get("user_id")
db = get_db_read_replica()
with db.scoped_session() as session:
followee_user_ids = session.query(Follow.followee_user_id).filter(
Follow.follower_user_id == current_user_id,
Follow.is_current == True,
Follow.is_delete == False,
)
followee_user_ids_subquery = followee_user_ids.subquery()
# Queries for tracks joined against followed users and counts
tracks_query = (
session.query(Track, ).join(
followee_user_ids_subquery,
Track.owner_id ==
followee_user_ids_subquery.c.followee_user_id,
).join(AggregateTrack, Track.track_id == AggregateTrack.track_id).
filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
# Query only tracks created `window` time ago (week, month, etc.)
Track.created_at >= text(f"NOW() - interval '1 {window}'"),
).order_by(
desc(AggregateTrack.repost_count + AggregateTrack.save_count),
desc(Track.track_id),
).limit(limit))
tracks_query_results = tracks_query.all()
tracks = helpers.query_result_to_list(tracks_query_results)
track_ids = list(map(lambda track: track["track_id"], tracks))
# Bundle peripheral info into track results
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
if args.get("with_users", False):
user_id_list = get_users_ids(tracks)
users = get_users_by_id(session, user_id_list)
for track in tracks:
user = users[track["owner_id"]]
if user:
track["user"] = user
return tracks
def get_top_followee_saves(saveType, args):
if saveType != 'track':
raise exceptions.ArgumentError(
"Invalid type provided, must be one of 'track'")
limit = args.get('limit', 25)
current_user_id = get_current_user_id()
db = get_db_read_replica()
with db.scoped_session() as session:
# Construct a subquery of all followees
followee_user_ids = (session.query(Follow.followee_user_id).filter(
Follow.follower_user_id == current_user_id,
Follow.is_current == True, Follow.is_delete == False))
followee_user_ids_subquery = followee_user_ids.subquery()
# Construct a subquery of all saves from followees aggregated by id
save_count = (session.query(
Save.save_item_id,
func.count(Save.save_item_id).label(
response_name_constants.save_count)).join(
followee_user_ids_subquery, Save.user_id ==
followee_user_ids_subquery.c.followee_user_id).filter(
Save.is_current == True,
Save.is_delete == False,
Save.save_type == saveType,
).group_by(Save.save_item_id).order_by(
desc(response_name_constants.save_count)).limit(limit))
save_count_subquery = save_count.subquery()
# Query for tracks joined against followee save counts
tracks_query = (session.query(Track, ).join(
save_count_subquery,
Track.track_id == save_count_subquery.c.save_item_id).filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
))
tracks_query_results = tracks_query.all()
tracks = helpers.query_result_to_list(tracks_query_results)
track_ids = list(map(lambda track: track['track_id'], tracks))
# bundle peripheral info into track results
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
if args.get('with_users', False):
user_id_list = get_users_ids(tracks)
users = get_users_by_id(session, user_id_list)
for track in tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return tracks
def get_playlist_tracks(args):
playlists = []
current_user_id = args.get("current_user_id")
limit = args.get("limit")
offset = args.get("offset")
db = get_db_read_replica()
with db.scoped_session() as session:
try:
playlist_id = args.get("playlist_id")
playlist = (
session
.query(Playlist)
.filter(
Playlist.is_current == True,
Playlist.playlist_id == playlist_id
)
.first()
)
if playlist is None:
return None
playlist_track_ids = [track_id['track']
for track_id in playlist.playlist_contents['track_ids']]
if limit and offset:
playlist_track_ids = playlist_track_ids[offset:offset+limit]
playlist_tracks = (
session
.query(Track)
.filter(
Track.is_current == True,
Track.track_id.in_(playlist_track_ids)
)
.all()
)
tracks = helpers.query_result_to_list(playlist_tracks)
tracks = populate_track_metadata(
session, playlist_track_ids, tracks, current_user_id)
if args.get("with_users", False):
add_users_to_tracks(session, tracks, current_user_id)
tracks_dict = {track['track_id']: track for track in tracks}
playlist_tracks = []
for track_id in playlist_track_ids:
playlist_tracks.append(tracks_dict[track_id])
return playlist_tracks
except sqlalchemy.orm.exc.NoResultFound:
pass
return playlists
def _get_user_listening_history(session: Session,
args: GetUserListeningHistoryArgs):
user_id = args["user_id"]
current_user_id = args["current_user_id"]
limit = args["limit"]
offset = args["offset"]
if user_id != current_user_id:
return []
listening_history_results = (session.query(
UserListeningHistory.listening_history).filter(
UserListeningHistory.user_id == current_user_id)).scalar()
if not listening_history_results:
return []
# add query pagination
listening_history_results = listening_history_results[offset:offset +
limit]
track_ids = []
listen_dates = []
for listen in listening_history_results:
track_ids.append(listen["track_id"])
listen_dates.append(listen["timestamp"])
track_results = (session.query(Track).filter(
Track.track_id.in_(track_ids))).all()
track_results_dict = {
track_result.track_id: track_result
for track_result in track_results
}
# sort tracks in listening history order
sorted_track_results = []
for track_id in track_ids:
if track_id in track_results_dict:
sorted_track_results.append(track_results_dict[track_id])
tracks = helpers.query_result_to_list(sorted_track_results)
# bundle peripheral info into track results
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
add_users_to_tracks(session, tracks, current_user_id)
for idx, track in enumerate(tracks):
track[response_name_constants.activity_timestamp] = listen_dates[idx]
return tracks
def get_trending_tracks(args):
limit, offset, current_user_id = args.get("limit"), args.get(
"offset"), args.get("current_user_id")
db = get_db_read_replica()
time = args.get('time')
query_time = None if time not in ["day", "week", "month", "year"] else time
with db.scoped_session() as session:
def get_unpopulated_trending():
trending_tracks = generate_trending(session, query_time,
args.get('genre', None), limit,
offset)
track_scores = [
z(time, track) for track in trending_tracks['listen_counts']
]
sorted_track_scores = sorted(track_scores,
key=lambda k: k['score'],
reverse=True)
track_ids = [track['track_id'] for track in sorted_track_scores]
tracks = get_unpopulated_tracks(session, track_ids)
return (tracks, track_ids)
# get scored trending tracks, either
# through cached redis value, or through `get_unpopulated_trending`
cache_keys = {"genre": args.get("genre"), "time": args.get("time")}
key = extract_key(f"generated-trending:{request.path}",
cache_keys.items())
(tracks, track_ids) = use_redis_cache(key, SCORES_CACHE_DURATION_SEC,
get_unpopulated_trending)
# populate track metadata
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return sorted_tracks
def get_tracks():
tracks = []
db = get_db()
with db.scoped_session() as session:
# Create initial query
base_query = session.query(Track)
base_query = base_query.filter(Track.is_current == True)
# Conditionally process an array of tracks
if "id" in request.args:
# Retrieve argument from flask request object
# Ensures empty parameters are not processed
track_id_str_list = request.args.getlist("id")
track_id_list = []
try:
track_id_list = [int(y) for y in track_id_str_list]
# Update query with track_id list
base_query = base_query.filter(
Track.track_id.in_(track_id_list))
except ValueError as e:
logger.error("Invalid value found in track id list",
exc_info=True)
raise e
# Allow filtering of tracks by a certain creator
if "user_id" in request.args:
user_id = request.args.get("user_id", type=int)
base_query = base_query.filter(Track.owner_id == user_id)
if "min_block_number" in request.args:
min_block_number = request.args.get("min_block_number", type=int)
base_query = base_query.filter(
Track.blocknumber >= min_block_number)
whitelist_params = [
'created_at', 'create_date', 'release_date', 'blocknumber',
'track_id'
]
base_query = parse_sort_param(base_query, Track, whitelist_params)
query_results = paginate_query(base_query).all()
tracks = helpers.query_result_to_list(query_results)
track_ids = list(map(lambda track: track["track_id"], tracks))
current_user_id = get_current_user_id(required=False)
# bundle peripheral info into track results
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
return api_helpers.success_response(tracks)
def search_track_tags(session, args):
"""
Gets the tracks with a given tag
Args:
session: sqlalchemy db session instance
args: dict of arguments
args.search_str: string the tag search string
args.current_user_id: id | null The user id making the query
args.limit: number the query limit of number of returns tracks
args.offset: number the query offset for results
Returns:
list of tracks sorted by play count
"""
track_ids = (session.query(TagTrackUserMatview.track_id).filter(
TagTrackUserMatview.tag == args["search_str"].lower()).all())
# track_ids is list of tuples - simplify to 1-D list
track_ids = [i[0] for i in track_ids]
tracks = (session.query(Track).filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
Track.track_id.in_(track_ids),
).all())
tracks = helpers.query_result_to_list(tracks)
track_play_counts = get_track_play_counts(session, track_ids)
tracks = populate_track_metadata(session, track_ids, tracks,
args["current_user_id"])
for track in tracks:
track_id = track["track_id"]
track[response_name_constants.play_count] = track_play_counts.get(
track_id, 0)
play_count_sorted_tracks = sorted(
tracks,
key=lambda i: i[response_name_constants.play_count],
reverse=True)
# Add pagination parameters to track and user results
play_count_sorted_tracks = play_count_sorted_tracks[slice(
args["offset"], args["offset"] + args["limit"], 1)]
return play_count_sorted_tracks
def get_save_tracks(args):
user_id = args.get("user_id")
current_user_id = args.get("current_user_id")
limit = args.get("limit")
offset = args.get("offset")
filter_deleted = args.get("filter_deleted")
db = get_db_read_replica()
with db.scoped_session() as session:
base_query = (session.query(Track, Save.created_at).join(
Save, Save.save_item_id == Track.track_id).filter(
Track.is_current == True,
Save.user_id == user_id,
Save.is_current == True,
Save.is_delete == False,
Save.save_type == SaveType.track,
))
# Allow filtering of deletes
if filter_deleted:
base_query = base_query.filter(Track.is_delete == False)
base_query = base_query.order_by(Save.created_at.desc(),
Track.track_id.desc())
query_results = add_query_pagination(base_query, limit, offset).all()
if not query_results:
return []
tracks, save_dates = zip(*query_results)
tracks = helpers.query_result_to_list(tracks)
track_ids = list(map(lambda track: track["track_id"], tracks))
# bundle peripheral info into track results
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
if args.get("with_users", False):
user_id_list = get_users_ids(tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in tracks:
user = users[track["owner_id"]]
if user:
track["user"] = user
for idx, track in enumerate(tracks):
track[response_name_constants.activity_timestamp] = save_dates[idx]
return tracks
def get_remix_track_parents(args):
"""Fetch remix parents for a given track.
Args:
args:dict
args.track_id: track id
args.limit: limit
args.offset: offset
args.with_users: with users
args.current_user_id: current user ID
"""
track_id = args.get("track_id")
current_user_id = args.get("current_user_id")
limit = args.get("limit")
offset = args.get("offset")
db = get_db_read_replica()
with db.scoped_session() as session:
def get_unpopulated_remix_parents():
base_query = (
session.query(Track)
.join(
Remix,
and_(
Remix.parent_track_id == Track.track_id,
Remix.child_track_id == track_id,
),
)
.filter(Track.is_current == True, Track.is_unlisted == False)
.order_by(desc(Track.created_at), desc(Track.track_id))
)
tracks = add_query_pagination(base_query, limit, offset).all()
tracks = helpers.query_result_to_list(tracks)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
key = make_cache_key(args)
(tracks, track_ids) = use_redis_cache(
key,
UNPOPULATED_REMIX_PARENTS_CACHE_DURATION_SEC,
get_unpopulated_remix_parents,
)
tracks = populate_track_metadata(session, track_ids, tracks, current_user_id)
if args.get("with_users", False):
add_users_to_tracks(session, tracks, current_user_id)
return tracks
def get_trending_tracks(args):
(limit, offset) = get_pagination_vars()
current_user_id = get_current_user_id(required=False)
db = get_db_read_replica()
time = args.get('time')
# Identity understands allTime as millennium.
# TODO: Change this in https://github.com/AudiusProject/audius-protocol/pull/768/files
query_time = time
if time == 'allTime':
query_time = 'millennium'
with db.scoped_session() as session:
trending_tracks = generate_trending(get_db_read_replica(), query_time,
args.get('genre', None), limit,
offset)
track_scores = [
z(time, track) for track in trending_tracks['listen_counts']
]
sorted_track_scores = sorted(track_scores,
key=lambda k: k['score'],
reverse=True)
track_ids = [track['track_id'] for track in sorted_track_scores]
tracks = session.query(Track).filter(
Track.is_current == True, Track.is_unlisted == False,
Track.stem_of == None, Track.track_id.in_(track_ids)).all()
tracks = helpers.query_result_to_list(tracks)
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return sorted_tracks
def get_trending_tracks(args):
(limit, offset) = get_pagination_vars()
current_user_id = get_current_user_id(required=False)
db = get_db_read_replica()
time = args.get('time')
with db.scoped_session() as session:
trending_tracks = generate_trending(get_db_read_replica(), time,
args.get('genre', None), limit,
offset)
track_scores = [
z(time, track) for track in trending_tracks['listen_counts']
]
sorted_track_scores = sorted(track_scores,
key=lambda k: k['score'],
reverse=True)
track_ids = [track['track_id'] for track in sorted_track_scores]
tracks = session.query(Track).filter(
Track.is_current == True, Track.is_unlisted == False,
Track.stem_of == None, Track.track_id.in_(track_ids)).all()
tracks = helpers.query_result_to_list(tracks)
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return sorted_tracks
def get_remix_track_parents(track_id, args):
db = get_db_read_replica()
with db.scoped_session() as session:
base_query = (session.query(Track).join(
Remix,
and_(Remix.parent_track_id == Track.track_id,
Remix.child_track_id == track_id)).filter(
Track.is_current == True,
Track.is_unlisted == False).order_by(
desc(Track.created_at), desc(Track.track_id)))
tracks = paginate_query(base_query).all()
tracks = helpers.query_result_to_list(tracks)
track_ids = list(map(lambda track: track["track_id"], tracks))
current_user_id = get_current_user_id(required=False)
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
if args.get("with_users", False):
add_users_to_tracks(session, tracks)
return tracks
def _get_trending_tracks_with_session(session: Session,
args: GetTrendingTracksArgs,
strategy: BaseTrendingStrategy):
current_user_id, genre, time = (
args.get("current_user_id"),
args.get("genre"),
args.get("time", "week"),
)
time_range = "week" if time not in ["week", "month", "year", "allTime"
] else time
key = make_trending_cache_key(time_range, genre, strategy.version)
# Will try to hit cached trending from task, falling back
# to generating it here if necessary and storing it with no TTL
(tracks, track_ids) = use_redis_cache(
key,
None,
make_generate_unpopulated_trending(session, genre, time_range,
strategy),
)
# populate track metadata
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track["track_id"]: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in sorted_tracks:
user = users[track["owner_id"]]
if user:
track["user"] = user
return sorted_tracks
def get_feed():
feed_results = []
db = get_db()
# Current user - user for whom feed is being generated
current_user_id = get_current_user_id()
with db.scoped_session() as session:
# Generate list of users followed by current user, i.e. 'followees'
followee_user_ids = (session.query(Follow.followee_user_id).filter(
Follow.follower_user_id == current_user_id,
Follow.is_current == True, Follow.is_delete == False).all())
followee_user_ids = [f[0] for f in followee_user_ids]
# Query playlists posted by followees, sorted and paginated by created_at desc
created_playlists_query = (session.query(Playlist).filter(
Playlist.is_current == True, Playlist.is_private == False,
Playlist.playlist_owner_id.in_(followee_user_ids)).order_by(
desc(Playlist.created_at)))
created_playlists = paginate_query(created_playlists_query,
False).all()
# get track ids for all tracks in playlists
playlist_track_ids = set()
for playlist in created_playlists:
for track in playlist.playlist_contents["track_ids"]:
playlist_track_ids.add(track["track"])
# get all track objects for track ids
playlist_tracks = (session.query(Track).filter(
Track.is_current == True,
Track.track_id.in_(playlist_track_ids)).all())
playlist_tracks_dict = {
track.track_id: track
for track in playlist_tracks
}
# get all track ids that have same owner as playlist and created in "same action"
# "same action": track created within [x time] before playlist creation
tracks_to_dedupe = set()
for playlist in created_playlists:
for track_entry in playlist.playlist_contents["track_ids"]:
track = playlist_tracks_dict.get(track_entry["track"])
if not track:
return api_helpers.error_response(
"Something caused the server to crash.")
max_timedelta = datetime.timedelta(
minutes=trackDedupeMaxMinutes)
if (track.owner_id == playlist.playlist_owner_id) and \
(track.created_at <= playlist.created_at) and \
(playlist.created_at - track.created_at <= max_timedelta):
tracks_to_dedupe.add(track.track_id)
tracks_to_dedupe = list(tracks_to_dedupe)
# Query tracks posted by followees, sorted & paginated by created_at desc
# exclude tracks that were posted in "same action" as playlist
created_tracks_query = (session.query(Track).filter(
Track.is_current == True, Track.owner_id.in_(followee_user_ids),
Track.track_id.notin_(tracks_to_dedupe)).order_by(
desc(Track.created_at)))
created_tracks = paginate_query(created_tracks_query, False).all()
# extract created_track_ids and created_playlist_ids
created_track_ids = [track.track_id for track in created_tracks]
created_playlist_ids = [
playlist.playlist_id for playlist in created_playlists
]
# query items reposted by followees, sorted by oldest followee repost of item;
# paginated by most recent repost timestamp
# exclude items also created by followees to guarantee order determinism
repost_subquery = (session.query(Repost).filter(
Repost.is_current == True, Repost.is_delete == False,
Repost.user_id.in_(followee_user_ids),
or_(
and_(Repost.repost_type == RepostType.track,
Repost.repost_item_id.notin_(created_track_ids)),
and_(Repost.repost_type == RepostType.track,
Repost.repost_item_id.notin_(
created_playlist_ids)))).subquery())
repost_query = (session.query(
repost_subquery.c.repost_item_id, repost_subquery.c.repost_type,
func.min(repost_subquery.c.created_at).label("min_created_at")
).group_by(
repost_subquery.c.repost_item_id,
repost_subquery.c.repost_type).order_by("min_created_at desc"))
followee_reposts = paginate_query(repost_query, False).all()
# build dict of track id -> oldest followee repost timestamp from followee_reposts above
track_repost_timestamp_dict = {}
playlist_repost_timestamp_dict = {}
for (repost_item_id, repost_type,
oldest_followee_repost_timestamp) in followee_reposts:
if repost_type == RepostType.track:
track_repost_timestamp_dict[
repost_item_id] = oldest_followee_repost_timestamp
elif repost_type in (RepostType.playlist, RepostType.album):
playlist_repost_timestamp_dict[
repost_item_id] = oldest_followee_repost_timestamp
# extract reposted_track_ids and reposted_playlist_ids
reposted_track_ids = list(track_repost_timestamp_dict.keys())
reposted_playlist_ids = list(playlist_repost_timestamp_dict.keys())
# Query tracks reposted by followees, excluding tracks already fetched from above
reposted_tracks = (session.query(Track).filter(
Track.is_current == True, Track.track_id.in_(reposted_track_ids),
Track.track_id.notin_(created_track_ids)).order_by(
desc(Track.created_at)).all())
# Query playlists reposted by followees, excluding playlists already fetched from above
reposted_playlists = (session.query(Playlist).filter(
Playlist.is_current == True, Playlist.is_private == False,
Playlist.playlist_id.in_(reposted_playlist_ids),
Playlist.playlist_id.notin_(created_playlist_ids)).all())
# Combine created + reposted track and playlist lists
tracks = helpers.query_result_to_list(created_tracks + reposted_tracks)
playlists = helpers.query_result_to_list(created_playlists +
reposted_playlists)
# define top level feed activity_timestamp to enable sorting
# activity_timestamp: created_at if item created by followee, else reposted_at
for track in tracks:
if track["owner_id"] in followee_user_ids:
track[response_name_constants.
activity_timestamp] = track["created_at"]
else:
track[response_name_constants.
activity_timestamp] = track_repost_timestamp_dict[
track["track_id"]]
for playlist in playlists:
if playlist["playlist_owner_id"] in followee_user_ids:
playlist[response_name_constants.
activity_timestamp] = playlist["created_at"]
else:
playlist[response_name_constants.activity_timestamp] = \
playlist_repost_timestamp_dict[playlist["playlist_id"]]
# bundle peripheral info into track and playlist objects
track_ids = list(map(lambda track: track["track_id"], tracks))
playlist_ids = list(
map(lambda playlist: playlist["playlist_id"], playlists))
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
playlists = populate_playlist_metadata(
session, playlist_ids, playlists,
[RepostType.playlist, RepostType.album],
[SaveType.playlist, SaveType.album], current_user_id)
# build combined feed of tracks and playlists
unsorted_feed = tracks + playlists
# sort feed based on activity_timestamp
sorted_feed = sorted(unsorted_feed,
key=lambda entry: entry[response_name_constants.
activity_timestamp],
reverse=True)
# truncate feed to requested limit
(limit, _) = get_pagination_vars()
feed_results = sorted_feed[0:limit]
return api_helpers.success_response(feed_results)
def track_search_query(
session,
search_str,
limit,
offset,
is_auto_complete,
current_user_id,
only_downloadable,
):
res = sqlalchemy.text(
# pylint: disable=C0301
f"""
select track_id, b.balance, b.associated_wallets_balance, u.is_saved from (
select distinct on (owner_id) track_id, owner_id, is_saved, total_score
from (
select track_id, owner_id, is_saved,
(
(:similarity_weight * sum(score)) +
(:title_weight * similarity(coalesce(title, ''), query)) +
(:user_name_weight * similarity(coalesce(user_name, ''), query)) +
(:repost_weight * log(case when (repost_count = 0) then 1 else repost_count end)) +
(case when (lower(query) = coalesce(title, '')) then :title_match_boost else 0 end) +
(case when (lower(query) = handle) then :handle_match_boost else 0 end) +
(case when (lower(query) = user_name) then :user_name_match_boost else 0 end)
{
'+ (case when (is_saved) then :current_user_saved_match_boost else 0 end)'
if current_user_id
else ""
}
) as total_score
from (
select
d."track_id" as track_id, d."word" as word, similarity(d."word", :query) as score,
d."track_title" as title, :query as query, d."user_name" as user_name, d."handle" as handle,
d."repost_count" as repost_count, d."owner_id" as owner_id
{
',s."user_id" is not null as is_saved'
if current_user_id
else ", false as is_saved"
}
from "track_lexeme_dict" d
{
"left outer join (select save_item_id, user_id from saves where saves.save_type = 'track' " +
"and saves.is_current = true " +
"and saves.is_delete = false and saves.user_id = :current_user_id )" +
" s on s.save_item_id = d.track_id"
if current_user_id
else ""
}
{
'inner join "tracks" t on t.track_id = d.track_id'
if only_downloadable
else ""
}
where (d."word" % lower(:query) or d."handle" = lower(:query) or d."user_name" % lower(:query))
{
"and (t.download->>'is_downloadable')::boolean is True"
if only_downloadable
else ""
}
) as results
group by track_id, title, query, user_name, handle, repost_count, owner_id, is_saved
) as results2
order by owner_id, total_score desc
) as u left join user_balances b on u.owner_id = b.user_id
order by total_score desc
limit :limit
offset :offset;
"""
)
track_result_proxy = session.execute(
res,
params={
"query": search_str,
"limit": limit,
"offset": offset,
"title_weight": search_title_weight,
"repost_weight": search_repost_weight,
"similarity_weight": search_similarity_weight,
"current_user_id": current_user_id,
"user_name_weight": search_user_name_weight,
"title_match_boost": search_title_exact_match_boost,
"handle_match_boost": search_handle_exact_match_boost,
"user_name_match_boost": search_user_name_exact_match_boost,
"current_user_saved_match_boost": current_user_saved_match_boost,
},
)
track_data = track_result_proxy.fetchall()
track_cols = track_result_proxy.keys()
# track_ids is list of tuples - simplify to 1-D list
track_ids = [track[track_cols.index("track_id")] for track in track_data]
saved_tracks = {
track[0] for track in track_data if track[track_cols.index("is_saved")]
}
tracks = get_unpopulated_tracks(session, track_ids, True)
# TODO: Populate track metadata should be sped up to be able to be
# used in search autocomplete as that'll give us better results.
if is_auto_complete:
# fetch users for tracks
track_owner_ids = list(map(lambda track: track["owner_id"], tracks))
users = get_unpopulated_users(session, track_owner_ids)
users_dict = {user["user_id"]: user for user in users}
# attach user objects to track objects
for i, track in enumerate(tracks):
user = users_dict[track["owner_id"]]
# Add user balance
balance = track_data[i][1]
associated_balance = track_data[i][2]
user[response_name_constants.balance] = balance
user[
response_name_constants.associated_wallets_balance
] = associated_balance
track["user"] = user
else:
# bundle peripheral info into track results
tracks = populate_track_metadata(session, track_ids, tracks, current_user_id)
# Preserve order from track_ids above
tracks_map = {}
for t in tracks:
tracks_map[t["track_id"]] = t
tracks = [tracks_map[track_id] for track_id in track_ids]
tracks_response = {
"all": tracks,
"saved": list(filter(lambda track: track["track_id"] in saved_tracks, tracks)),
}
return tracks_response
def get_remixes_of(args):
track_id = args.get("track_id")
current_user_id = args.get("current_user_id")
limit, offset = args.get("limit"), args.get("offset")
db = get_db_read_replica()
with db.scoped_session() as session:
def get_unpopulated_remixes():
# Fetch the parent track to get the track's owner id
parent_track_res = get_unpopulated_tracks(session, [track_id],
False, False)
if not parent_track_res or parent_track_res[0] is None:
raise exceptions.ArgumentError("Invalid track_id provided")
parent_track = parent_track_res[0]
track_owner_id = parent_track['owner_id']
# Create subquery for save counts for sorting
save_count_subquery = create_save_count_subquery(
session, SaveType.track)
# Create subquery for repost counts for sorting
repost_count_subquery = create_repost_count_subquery(
session, RepostType.track)
# Get the 'children' remix tracks
# Use the track owner id to fetch reposted/saved tracks returned first
base_query = (
session.query(
Track
)
.join(
Remix,
and_(
Remix.child_track_id == Track.track_id,
Remix.parent_track_id == track_id
)
).outerjoin(
Save,
and_(
Save.save_item_id == Track.track_id,
Save.save_type == SaveType.track,
Save.is_current == True,
Save.is_delete == False,
Save.user_id == track_owner_id
)
).outerjoin(
Repost,
and_(
Repost.repost_item_id == Track.track_id,
Repost.user_id == track_owner_id,
Repost.repost_type == RepostType.track,
Repost.is_current == True,
Repost.is_delete == False
)
).outerjoin(
repost_count_subquery,
repost_count_subquery.c['id'] == Track.track_id
).outerjoin(
save_count_subquery,
save_count_subquery.c['id'] == Track.track_id
)
.filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False
)
# 1. Co-signed tracks ordered by save + repost count
# 2. Other tracks ordered by save + repost count
.order_by(
desc(
# If there is no "co-sign" for the track (no repost or save from the parent owner),
# defer to secondary sort
case(
[
(and_(Repost.created_at == None,
Save.created_at == None), 0),
],
else_=(
func.coalesce(repost_count_subquery.c.repost_count, 0) + \
func.coalesce(save_count_subquery.c.save_count, 0)
)
)
),
# Order by saves + reposts
desc(
func.coalesce(repost_count_subquery.c.repost_count, 0) + \
func.coalesce(save_count_subquery.c.save_count, 0)
),
# Ties, pick latest track id
desc(Track.track_id)
)
)
(tracks, count) = add_query_pagination(base_query, limit, offset,
True, True)
tracks = tracks.all()
tracks = helpers.query_result_to_list(tracks)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids, count)
key = make_cache_key(args)
(tracks, track_ids,
count) = use_redis_cache(key, UNPOPULATED_REMIXES_CACHE_DURATION_SEC,
get_unpopulated_remixes)
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
if args.get("with_users", False):
add_users_to_tracks(session, tracks, current_user_id)
return {'tracks': tracks, 'count': count}
def get_remixable_tracks(args):
"""Gets a list of remixable tracks"""
db = get_db_read_replica()
limit = args.get("limit", 25)
current_user_id = args.get("current_user_id", None)
StemTrack = aliased(Track)
with db.scoped_session() as session:
# Subquery to get current tracks that have stems
remixable_tracks_subquery = (session.query(Track).join(
Stem, Stem.parent_track_id == Track.track_id).join(
StemTrack, Stem.child_track_id == StemTrack.track_id).filter(
Track.is_current == True,
Track.is_unlisted == False,
Track.is_delete == False,
StemTrack.is_current == True,
StemTrack.is_unlisted == False,
StemTrack.is_delete == False,
).distinct(Track.track_id).subquery())
track_alias = aliased(Track, remixable_tracks_subquery)
count_subquery = session.query(
AggregateTrack.track_id.label("id"),
(AggregateTrack.repost_count +
AggregateTrack.save_count).label("count"),
).subquery()
query = (session.query(
track_alias,
count_subquery.c["count"],
decayed_score(count_subquery.c["count"],
track_alias.created_at).label("score"),
).join(
count_subquery,
count_subquery.c["id"] == track_alias.track_id,
).order_by(desc("score"), desc(track_alias.track_id)).limit(limit))
results = query.all()
tracks = []
for result in results:
track = result[0]
score = result[-1]
track = helpers.model_to_dictionary(track)
track["score"] = score
tracks.append(track)
track_ids = list(map(lambda track: track["track_id"], tracks))
# Get user specific data for tracks
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
if args.get("with_users", False):
add_users_to_tracks(session, tracks, current_user_id)
else:
# Remove the user from the tracks
tracks = [{key: val
for key, val in dict.items() if key != "user"}
for dict in tracks]
return tracks
def get_trending_playlists(args, strategy):
"""Returns Trending Playlists. Checks Redis cache for unpopulated playlists."""
db = get_db_read_replica()
with db.scoped_session() as session:
current_user_id = args.get("current_user_id", None)
with_tracks = args.get("with_tracks", False)
time = args.get("time")
limit, offset = args.get("limit"), args.get("offset")
key = make_trending_cache_key(time, strategy.version)
# Get unpopulated playlists,
# cached if it exists.
(playlists, playlist_ids) = use_redis_cache(
key, None, make_get_unpopulated_playlists(session, time, strategy))
# Apply limit + offset early to reduce the amount of
# population work we have to do
if limit is not None and offset is not None:
playlists = playlists[offset:limit + offset]
playlist_ids = playlist_ids[offset:limit + offset]
# Populate playlist metadata
playlists = populate_playlist_metadata(
session, playlist_ids, playlists,
[RepostType.playlist, RepostType.album],
[SaveType.playlist, SaveType.album], current_user_id)
trimmed_track_ids = None
for playlist in playlists:
playlist["track_count"] = len(playlist["tracks"])
playlist["tracks"] = playlist["tracks"][:PLAYLIST_TRACKS_LIMIT]
# Trim track_ids, which ultimately become added_timestamps
# and need to match the tracks.
trimmed_track_ids = {
track["track_id"]
for track in playlist["tracks"]
}
playlist_track_ids = playlist["playlist_contents"]["track_ids"]
playlist_track_ids = list(
filter(lambda track_id: track_id["track"] in trimmed_track_ids,
playlist_track_ids))
playlist["playlist_contents"]["track_ids"] = playlist_track_ids
playlists_map = {
playlist['playlist_id']: playlist
for playlist in playlists
}
if with_tracks:
# populate track metadata
tracks = []
for playlist in playlists:
playlist_tracks = playlist["tracks"]
tracks.extend(playlist_tracks)
track_ids = [track["track_id"] for track in tracks]
populated_tracks = populate_track_metadata(session, track_ids,
tracks, current_user_id)
# Add users if necessary
add_users_to_tracks(session, populated_tracks, current_user_id)
# Re-associate tracks with playlists
# track_id -> populated_track
populated_track_map = {
track["track_id"]: track
for track in populated_tracks
}
for playlist in playlists_map.values():
for i in range(len(playlist["tracks"])):
track_id = playlist["tracks"][i]["track_id"]
populated = populated_track_map[track_id]
playlist["tracks"][i] = populated
playlist["tracks"] = list(map(extend_track,
playlist["tracks"]))
# re-sort playlists to original order, because populate_playlist_metadata
# unsorts.
sorted_playlists = [
playlists_map[playlist_id] for playlist_id in playlist_ids
]
# Add users to playlists
user_id_list = get_users_ids(sorted_playlists)
users = get_users_by_id(session, user_id_list, current_user_id)
for playlist in sorted_playlists:
user = users[playlist['playlist_owner_id']]
if user:
playlist['user'] = user
# Extend the playlists
playlists = list(map(extend_playlist, playlists))
return sorted_playlists
def test_populate_track_metadata(app):
"""Tests that populate_track_metadata works after aggregate_user refresh"""
with app.app_context():
db = get_db()
test_entities = {
'tracks': [{
"track_id": 1,
"owner_id": 1
}, {
"track_id": 2,
"owner_id": 1
}, {
"track_id": 3,
"owner_id": 2
}, {
"track_id": 4,
"owner_id": 2
}, {
"track_id": 5,
"owner_id": 2
}, {
"track_id": 6,
"owner_id": 2
}, {
"track_id": 7,
"owner_id": 3
}, {
"track_id": 8,
"owner_id": 3
}, {
"track_id": 9,
"owner_id": 3
}, {
"track_id": 10,
"is_unlisted": True,
"owner_id": 3
}],
'users': [{
'user_id': 1,
'handle': 'user1'
}, {
'user_id': 2,
'handle': 'user2'
}, {
'user_id': 3,
'handle': 'user3'
}, {
'user_id': 4,
'handle': 'user4'
}],
'reposts': [{
"repost_item_id": 1,
"repost_type": 'track',
"user_id": 2
}, {
"repost_item_id": 1,
"repost_type": 'track',
"user_id": 3
}, {
"repost_item_id": 2,
"repost_type": 'track',
"user_id": 1
}],
'saves': [{
"save_item_id": 1,
"save_type": 'track',
"user_id": 2
}, {
"save_item_id": 1,
"save_type": 'track',
"user_id": 3
}, {
"save_item_id": 3,
"save_type": 'track',
"user_id": 2
}, {
"save_item_id": 3,
"save_type": 'track',
"user_id": 1
}],
'follows': [{
"follower_user_id": 1,
"followee_user_id": 2
}, {
"follower_user_id": 1,
"followee_user_id": 3
}]
}
populate_mock_db(db, test_entities)
with db.scoped_session() as session:
session.execute("REFRESH MATERIALIZED VIEW aggregate_track")
track_ids = [1, 2, 3]
tracks = [
{
"track_id": 1
},
{
"track_id": 2
},
{
"track_id": 3
},
]
tracks = populate_track_metadata(session, track_ids, tracks, None)
assert len(tracks) == 3
assert tracks[0]['track_id'] == 1
assert tracks[0][response_name_constants.repost_count] == 2
assert tracks[0][response_name_constants.save_count] == 2
assert tracks[0][response_name_constants.play_count] == 0
assert tracks[1]['track_id'] == 2
assert tracks[1][response_name_constants.repost_count] == 1
assert tracks[1][response_name_constants.save_count] == 0
assert tracks[1][response_name_constants.play_count] == 0
assert tracks[2]['track_id'] == 3
assert tracks[2][response_name_constants.repost_count] == 0
assert tracks[2][response_name_constants.save_count] == 2
assert tracks[2][response_name_constants.play_count] == 0
curr_track_ids = [1, 2, 3]
curr_tracks = [{"track_id": 1}, {"track_id": 2}, {"track_id": 3}]
tracks = populate_track_metadata(session, curr_track_ids, curr_tracks,
1)
assert len(tracks) == 3
assert tracks[0]['track_id'] == 1
repost_user_ids = [
repost['user_id']
for repost in tracks[0][response_name_constants.followee_reposts]
]
repost_user_ids.sort()
assert repost_user_ids == [2, 3]
save_user_ids = [
save['user_id']
for save in tracks[0][response_name_constants.followee_saves]
]
save_user_ids.sort()
assert save_user_ids == [2, 3]
assert tracks[0][
response_name_constants.has_current_user_reposted] == False
assert tracks[0][
response_name_constants.has_current_user_saved] == False
assert tracks[1]['track_id'] == 2
assert tracks[1][response_name_constants.followee_reposts] == []
assert tracks[1][response_name_constants.followee_saves] == []
assert tracks[1][
response_name_constants.has_current_user_reposted] == True
assert tracks[1][
response_name_constants.has_current_user_saved] == False
assert tracks[2]['track_id'] == 3
assert tracks[2][response_name_constants.followee_reposts] == []
save_user_ids = [
save['user_id']
for save in tracks[2][response_name_constants.followee_saves]
]
save_user_ids.sort()
assert save_user_ids == [2]
assert tracks[2][
response_name_constants.has_current_user_reposted] == False
assert tracks[2][
response_name_constants.has_current_user_saved] == True
def get_playlist_tracks(session, args):
"""Accepts args:
{
# optionally pass in full playlists to avoid having to fetch
"playlists": Playlist[]
# not needed if playlists are passed
"playlist_ids": string[]
"current_user_id": int
"populate_tracks": boolean # whether to add users & metadata to tracks
}
Returns: {
playlist_id: Playlist
}
"""
try:
playlists = args.get("playlists")
if not playlists:
playlist_ids = args.get("playlist_ids", [])
playlists = (session.query(Playlist).filter(
Playlist.is_current == True,
Playlist.playlist_id.in_(playlist_ids)))
playlists = list(map(helpers.model_to_dictionary, playlists))
if not playlists:
return {}
# track_id -> [playlist_id]
track_ids_set = set()
for playlist in playlists:
playlist_id = playlist['playlist_id']
for track_id_dict in playlist['playlist_contents']['track_ids']:
track_id = track_id_dict['track']
track_ids_set.add(track_id)
playlist_tracks = (session.query(Track).filter(
Track.is_current == True,
Track.track_id.in_(list(track_ids_set))).all())
tracks = helpers.query_result_to_list(playlist_tracks)
if args.get("populate_tracks"):
current_user_id = args.get("current_user_id")
tracks = populate_track_metadata(session, list(track_ids_set),
tracks, current_user_id)
add_users_to_tracks(session, tracks, current_user_id)
# { track_id => track }
track_ids_map = {track["track_id"]: track for track in tracks}
# { playlist_id => [track]}
playlists_map = {}
for playlist in playlists:
playlist_id = playlist["playlist_id"]
playlists_map[playlist_id] = []
for track_id_dict in playlist['playlist_contents']['track_ids']:
track_id = track_id_dict['track']
track = track_ids_map[track_id]
playlists_map[playlist_id].append(track)
return playlists_map
except sqlalchemy.orm.exc.NoResultFound:
return {}
def track_search_query(session, searchStr, limit, offset, personalized,
is_auto_complete, current_user_id):
if personalized and not current_user_id:
return []
res = sqlalchemy.text(
# pylint: disable=C0301
f"""
select track_id from (
select track_id, (sum(score) + (:title_weight * similarity(coalesce(title, ''), query))) as total_score from (
select
d."track_id" as track_id, d."word" as word, similarity(d."word", :query) as score,
d."track_title" as title, :query as query
from "track_lexeme_dict" d
{
'inner join "saves" s on s.save_item_id = d.track_id'
if personalized and current_user_id
else ""
}
where d."word" % :query
{
"and s.save_type='track' and s.is_current=true and s.is_delete=false and s.user_id = :current_user_id"
if personalized and current_user_id
else ""
}
) as results
group by track_id, title, query
) as results2
order by total_score desc, track_id asc
limit :limit
offset :offset;
""")
track_ids = session.execute(
res,
{
"query": searchStr,
"limit": limit,
"offset": offset,
"title_weight": trackTitleWeight,
"current_user_id": current_user_id
},
).fetchall()
# track_ids is list of tuples - simplify to 1-D list
track_ids = [i[0] for i in track_ids]
tracks = (session.query(Track).filter(
Track.is_delete == False,
Track.is_current == True,
Track.is_unlisted == False,
Track.stem_of == None,
Track.track_id.in_(track_ids),
).all())
tracks = helpers.query_result_to_list(tracks)
if is_auto_complete == True:
# fetch users for tracks
track_owner_ids = list(map(lambda track: track["owner_id"], tracks))
users = (session.query(User).filter(
User.is_current == True, User.user_id.in_(track_owner_ids)).all())
users = helpers.query_result_to_list(users)
users_dict = {user["user_id"]: user for user in users}
# attach user objects to track objects
for track in tracks:
track["user"] = users_dict[track["owner_id"]]
else:
# bundle peripheral info into track results
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
# preserve order from track_ids above
tracks = [
next(t for t in tracks if t["track_id"] == track_id)
for track_id in track_ids
]
return tracks
def get_tracks(args):
"""
Gets tracks.
A note on caching strategy:
- This method is cached at two layers: at the API via the @cache decorator,
and within this method using the shared get_unpopulated_tracks cache.
The shared cache only works when fetching via ID, so calls to fetch tracks
via handle, asc/desc sort, or filtering by block_number won't hit the shared cache.
These will hit the API cache unless they have a current_user_id included.
"""
tracks = []
db = get_db_read_replica()
with db.scoped_session() as session:
def get_tracks_and_ids():
if "handle" in args:
handle = args.get("handle")
user_id = session.query(User.user_id).filter(
User.handle_lc == handle.lower()).first()
args["user_id"] = user_id
can_use_shared_cache = ("id" in args
and not "min_block_number" in args
and not "sort" in args
and not "user_id" in args)
if can_use_shared_cache:
should_filter_deleted = args.get("filter_deleted", False)
tracks = get_unpopulated_tracks(session, args["id"],
should_filter_deleted)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
(limit, offset) = get_pagination_vars()
args["limit"] = limit
args["offset"] = offset
tracks = _get_tracks(session, args)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
(tracks, track_ids) = get_tracks_and_ids()
# bundle peripheral info into track results
current_user_id = args.get("current_user_id")
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
if args.get("with_users", False):
user_id_list = get_users_ids(tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return tracks
def get_tracks(args):
"""
Gets tracks.
A note on caching strategy:
- This method is cached at two layers: at the API via the @cache decorator,
and within this method using the shared get_unpopulated_tracks cache.
The shared cache only works when fetching via ID, so calls to fetch tracks
via handle, asc/desc sort, or filtering by block_number won't hit the shared cache.
These will hit the API cache unless they have a current_user_id included.
"""
tracks = []
db = get_db_read_replica()
with db.scoped_session() as session:
def get_tracks_and_ids():
if "handle" in args:
handle = args.get("handle")
user_id = session.query(User.user_id).filter(
User.handle_lc == handle.lower()).first()
args["user_id"] = user_id
can_use_shared_cache = ("id" in args
and not "min_block_number" in args
and not "sort" in args
and not "user_id" in args)
if can_use_shared_cache:
should_filter_deleted = args.get("filter_deleted", False)
tracks = get_unpopulated_tracks(session, args["id"],
should_filter_deleted)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
# Create initial query
base_query = session.query(Track)
base_query = base_query.filter(Track.is_current == True,
Track.is_unlisted == False,
Track.stem_of == None)
# Conditionally process an array of tracks
if "id" in args:
track_id_list = args.get("id")
try:
# Update query with track_id list
base_query = base_query.filter(
Track.track_id.in_(track_id_list))
except ValueError as e:
logger.error("Invalid value found in track id list",
exc_info=True)
raise e
# Allow filtering of tracks by a certain creator
if "user_id" in args:
user_id = args.get("user_id")
base_query = base_query.filter(Track.owner_id == user_id)
# Allow filtering of deletes
if "filter_deleted" in args:
filter_deleted = args.get("filter_deleted")
if filter_deleted:
base_query = base_query.filter(Track.is_delete == False)
if "min_block_number" in args:
min_block_number = args.get("min_block_number")
base_query = base_query.filter(
Track.blocknumber >= min_block_number)
if "sort" in args:
if args["sort"] == "date":
base_query = base_query.order_by(Track.created_at.desc(),
Track.track_id.desc())
elif args["sort"] == "plays":
base_query = base_query.join(
AggregatePlays, AggregatePlays.play_item_id ==
Track.track_id).order_by(AggregatePlays.count.desc())
else:
whitelist_params = [
'created_at', 'create_date', 'release_date',
'blocknumber', 'track_id'
]
base_query = parse_sort_param(base_query, Track,
whitelist_params)
query_results = paginate_query(base_query).all()
tracks = helpers.query_result_to_list(query_results)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
(tracks, track_ids) = get_tracks_and_ids()
# bundle peripheral info into track results
current_user_id = args.get("current_user_id")
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
if args.get("with_users", False):
user_id_list = get_users_ids(tracks)
users = get_users_by_id(session, user_id_list, current_user_id)
for track in tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return tracks
def search_tags():
search_str = request.args.get("query", type=str)
current_user_id = get_current_user_id(required=False)
if not search_str:
raise exceptions.ArgumentError("Invalid value for parameter 'query'")
user_tag_count = request.args.get("user_tag_count", type=str)
if not user_tag_count:
user_tag_count = "2"
kind = request.args.get("kind", type=str, default="all")
validSearchKinds = [SearchKind.all, SearchKind.tracks, SearchKind.users]
try:
searchKind = SearchKind[kind]
if searchKind not in validSearchKinds:
raise Exception
except Exception:
return api_helpers.error_response(
"Invalid value for parameter 'kind' must be in %s" %
[k.name for k in validSearchKinds], 400)
results = {}
(limit, offset) = get_pagination_vars()
like_tags_str = str.format('%{}%', search_str)
db = get_db_read_replica()
with db.scoped_session() as session:
if (searchKind in [SearchKind.all, SearchKind.tracks]):
track_res = sqlalchemy.text(f"""
select distinct(track_id)
from
(
select
strip(to_tsvector(tracks.tags)) as tagstrip,
track_id
from
tracks
where
(tags like :like_tags_query)
and (is_current is true)
and (is_delete is false)
and (is_unlisted is false)
and (stem_of is NULL)
order by
updated_at desc
) as t
where
tagstrip @@ to_tsquery(:query);
""")
track_ids = session.execute(track_res, {
"query": search_str,
"like_tags_query": like_tags_str
}).fetchall()
# track_ids is list of tuples - simplify to 1-D list
track_ids = [i[0] for i in track_ids]
tracks = (session.query(Track).filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
Track.track_id.in_(track_ids),
).all())
tracks = helpers.query_result_to_list(tracks)
track_play_counts = get_track_play_counts(track_ids)
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
for track in tracks:
track_id = track["track_id"]
track[response_name_constants.
play_count] = track_play_counts.get(track_id, 0)
play_count_sorted_tracks = \
sorted(
tracks, key=lambda i: i[response_name_constants.play_count], reverse=True)
# Add pagination parameters to track and user results
play_count_sorted_tracks = \
play_count_sorted_tracks[slice(offset, offset + limit, 1)]
results['tracks'] = play_count_sorted_tracks
if (searchKind in [SearchKind.all, SearchKind.users]):
user_res = sqlalchemy.text(f"""
select * from
(
select
count(track_id),
owner_id
from
(
select
strip(to_tsvector(tracks.tags)) as tagstrip,
track_id,
owner_id
from
tracks
where
(tags like :like_tags_query)
and (is_current is true)
and (is_unlisted is false)
and (stem_of is NULL)
order by
updated_at desc
) as t
where
tagstrip @@ to_tsquery(:query)
group by
owner_id
order by
count desc
) as usr
where
usr.count >= :user_tag_count;
""")
user_ids = session.execute(
user_res, {
"query": search_str,
"like_tags_query": like_tags_str,
"user_tag_count": user_tag_count
}).fetchall()
# user_ids is list of tuples - simplify to 1-D list
user_ids = [i[1] for i in user_ids]
users = (session.query(User).filter(
User.is_current == True, User.user_id.in_(user_ids)).all())
users = helpers.query_result_to_list(users)
users = populate_user_metadata(session, user_ids, users,
current_user_id)
followee_sorted_users = \
sorted(
users, key=lambda i: i[response_name_constants.follower_count], reverse=True)
followee_sorted_users = \
followee_sorted_users[slice(offset, offset + limit, 1)]
results['users'] = followee_sorted_users
# Add personalized results for a given user
if current_user_id:
if (searchKind in [SearchKind.all, SearchKind.tracks]):
# Query saved tracks for the current user that contain this tag
saves_query = (session.query(Save.save_item_id).filter(
Save.is_current == True, Save.is_delete == False,
Save.save_type == SaveType.track,
Save.user_id == current_user_id,
Save.save_item_id.in_(track_ids)).all())
saved_track_ids = [i[0] for i in saves_query]
saved_tracks = (session.query(Track).filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
Track.track_id.in_(saved_track_ids),
).all())
saved_tracks = helpers.query_result_to_list(saved_tracks)
for saved_track in saved_tracks:
saved_track_id = saved_track["track_id"]
saved_track[response_name_constants.play_count] = \
track_play_counts.get(saved_track_id, 0)
saved_tracks = \
populate_track_metadata(
session, saved_track_ids, saved_tracks, current_user_id)
# Sort and paginate
play_count_sorted_saved_tracks = \
sorted(
saved_tracks, key=lambda i: i[response_name_constants.play_count], reverse=True)
play_count_sorted_saved_tracks = \
play_count_sorted_saved_tracks[slice(
offset, offset + limit, 1)]
results['saved_tracks'] = play_count_sorted_saved_tracks
if (searchKind in [SearchKind.all, SearchKind.users]):
# Query followed users that have referenced this tag
followed_user_query = (session.query(
Follow.followee_user_id).filter(
Follow.is_current == True, Follow.is_delete == False,
Follow.follower_user_id == current_user_id,
Follow.followee_user_id.in_(user_ids)).all())
followed_user_ids = [i[0] for i in followed_user_query]
followed_users = (session.query(User).filter(
User.is_current == True,
User.user_id.in_(followed_user_ids)).all())
followed_users = helpers.query_result_to_list(followed_users)
followed_users = \
populate_user_metadata(
session,
followed_user_ids,
followed_users,
current_user_id
)
followed_users_followee_sorted = \
sorted(
followed_users,
key=lambda i: i[response_name_constants.follower_count],
reverse=True)
followed_users_followee_sorted = \
followed_users_followee_sorted[slice(
offset, offset + limit, 1)]
results['followed_users'] = followed_users_followee_sorted
return api_helpers.success_response(results)
def track_search_query(session, searchStr, limit, offset, personalized,
is_auto_complete, current_user_id, only_downloadable):
if personalized and not current_user_id:
return []
res = sqlalchemy.text(
# pylint: disable=C0301
f"""
select track_id from (
select
track_id,
(sum(score) + (:title_weight * similarity(coalesce(title, ''), query))) as total_score
from (
select
d."track_id" as track_id, d."word" as word, similarity(d."word", :query) as score,
d."track_title" as title, :query as query
from "track_lexeme_dict" d
{
'inner join "saves" s on s.save_item_id = d.track_id'
if personalized and current_user_id
else ""
}
{
'inner join "tracks" t on t.track_id = d.track_id'
if only_downloadable
else ""
}
where d."word" % :query
{
"and s.save_type='track' and s.is_current=true and " +
"s.is_delete=false and s.user_id = :current_user_id"
if personalized and current_user_id
else ""
}
{
"and (t.download->>'is_downloadable')::boolean is True"
if only_downloadable
else ""
}
) as results
group by track_id, title, query
) as results2
order by total_score desc, track_id asc
limit :limit
offset :offset;
""")
track_ids = session.execute(
res,
{
"query": searchStr,
"limit": max(limit, MIN_SEARCH_LEXEME_LIMIT),
"offset": offset,
"title_weight": trackTitleWeight,
"current_user_id": current_user_id
},
).fetchall()
# track_ids is list of tuples - simplify to 1-D list
track_ids = [i[0] for i in track_ids]
tracks = get_unpopulated_tracks(session, track_ids, True)
# TODO: Populate track metadata should be sped up to be able to be
# used in search autocomplete as that'll give us better results.
if is_auto_complete == True:
# fetch users for tracks
track_owner_ids = list(map(lambda track: track["owner_id"], tracks))
users = get_unpopulated_users(session, track_owner_ids)
users_dict = {user["user_id"]: user for user in users}
# attach user objects to track objects
for track in tracks:
track["user"] = users_dict[track["owner_id"]]
tracks = populate_track_repost_counts(session, track_ids, tracks)
else:
# bundle peripheral info into track results
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
# Preserve order from track_ids above
tracks_map = {}
for t in tracks:
tracks_map[t["track_id"]] = t
tracks = [tracks_map[track_id] for track_id in track_ids]
# Sort tracks by extra criteria for "best match"
tracks.sort(key=cmp_to_key(compare_tracks))
return tracks[0:limit]
def get_repost_feed_for_user(user_id, args):
feed_results = {}
db = get_db_read_replica()
current_user_id = args.get("current_user_id")
with db.scoped_session() as session:
if "handle" in args:
handle = args.get("handle")
user_id = session.query(
User.user_id).filter(User.handle_lc == handle.lower()).first()
# query all reposts by user
repost_query = (session.query(Repost).filter(
Repost.is_current == True, Repost.is_delete == False,
Repost.user_id == user_id).order_by(desc(Repost.created_at),
desc(Repost.repost_item_id),
desc(Repost.repost_type)))
reposts = paginate_query(repost_query).all()
# get track reposts from above
track_reposts = [
r for r in reposts if r.repost_type == RepostType.track
]
# get reposted track ids
repost_track_ids = [r.repost_item_id for r in track_reposts]
# get playlist reposts from above
playlist_reposts = [
r for r in reposts if r.repost_type == RepostType.playlist
or r.repost_type == RepostType.album
]
# get reposted playlist ids
repost_playlist_ids = [r.repost_item_id for r in playlist_reposts]
track_reposts = helpers.query_result_to_list(track_reposts)
playlist_reposts = helpers.query_result_to_list(playlist_reposts)
# build track/playlist id --> repost dict from repost lists
track_repost_dict = {
repost["repost_item_id"]: repost
for repost in track_reposts
}
playlist_repost_dict = {
repost["repost_item_id"]: repost
for repost in playlist_reposts
}
# query tracks for repost_track_ids
track_query = (session.query(Track).filter(
Track.is_current == True, Track.is_delete == False,
Track.is_unlisted == False, Track.stem_of == None,
Track.track_id.in_(repost_track_ids)).order_by(
desc(Track.created_at)))
tracks = track_query.all()
tracks = helpers.query_result_to_list(tracks)
# get track ids
track_ids = [track["track_id"] for track in tracks]
# query playlists for repost_playlist_ids
playlist_query = (session.query(Playlist).filter(
Playlist.is_current == True, Playlist.is_delete == False,
Playlist.is_private == False,
Playlist.playlist_id.in_(repost_playlist_ids)).order_by(
desc(Playlist.created_at)))
playlists = playlist_query.all()
playlists = helpers.query_result_to_list(playlists)
# get playlist ids
playlist_ids = [playlist["playlist_id"] for playlist in playlists]
# populate full metadata
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
playlists = populate_playlist_metadata(
session, playlist_ids, playlists,
[RepostType.playlist, RepostType.album],
[SaveType.playlist, SaveType.album], current_user_id)
# add activity timestamps
for track in tracks:
track[response_name_constants.
activity_timestamp] = track_repost_dict[
track["track_id"]]["created_at"]
for playlist in playlists:
playlist[response_name_constants.activity_timestamp] = \
playlist_repost_dict[playlist["playlist_id"]]["created_at"]
unsorted_feed = tracks + playlists
# sort feed by repost timestamp desc
feed_results = sorted(unsorted_feed,
key=lambda entry: entry[response_name_constants.
activity_timestamp],
reverse=True)
if args.get("with_users", False):
user_id_list = get_users_ids(feed_results)
users = get_users_by_id(session, user_id_list)
for result in feed_results:
if 'playlist_owner_id' in result:
user = users[result['playlist_owner_id']]
if user:
result['user'] = user
elif 'owner_id' in result:
user = users[result['owner_id']]
if user:
result['user'] = user
return feed_results
def search_tags():
search_str = request.args.get("query", type=str)
current_user_id = get_current_user_id(required=False)
if not search_str:
raise exceptions.ArgumentError("Invalid value for parameter 'query'")
user_tag_count = request.args.get("user_tag_count", type=str)
if not user_tag_count:
user_tag_count = "2"
kind = request.args.get("kind", type=str, default="all")
validSearchKinds = [SearchKind.all, SearchKind.tracks, SearchKind.users]
try:
searchKind = SearchKind[kind]
if searchKind not in validSearchKinds:
raise Exception
except Exception:
return api_helpers.error_response(
"Invalid value for parameter 'kind' must be in %s" %
[k.name for k in validSearchKinds], 400)
results = {}
(limit, offset) = get_pagination_vars()
db = get_db_read_replica()
with db.scoped_session() as session:
if (searchKind in [SearchKind.all, SearchKind.tracks]):
results['tracks'] = search_track_tags(
session, {
'search_str': search_str,
'current_user_id': current_user_id,
'limit': limit,
'offset': offset
})
if (searchKind in [SearchKind.all, SearchKind.users]):
results['users'] = search_user_tags(
session, {
'search_str': search_str,
'current_user_id': current_user_id,
"user_tag_count": user_tag_count,
'limit': limit,
'offset': offset
})
# Add personalized results for a given user
if current_user_id:
if (searchKind in [SearchKind.all, SearchKind.tracks]):
# Query saved tracks for the current user that contain this tag
track_ids = [track['track_id'] for track in results['tracks']]
track_play_counts = {
track['track_id']: track[response_name_constants.play_count]
for track in results['tracks']
}
saves_query = (session.query(Save.save_item_id).filter(
Save.is_current == True, Save.is_delete == False,
Save.save_type == SaveType.track,
Save.user_id == current_user_id,
Save.save_item_id.in_(track_ids)).all())
saved_track_ids = [i[0] for i in saves_query]
saved_tracks = (session.query(Track).filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
Track.track_id.in_(saved_track_ids),
).all())
saved_tracks = helpers.query_result_to_list(saved_tracks)
for saved_track in saved_tracks:
saved_track_id = saved_track["track_id"]
saved_track[response_name_constants.play_count] = \
track_play_counts.get(saved_track_id, 0)
saved_tracks = \
populate_track_metadata(
session, saved_track_ids, saved_tracks, current_user_id)
# Sort and paginate
play_count_sorted_saved_tracks = \
sorted(
saved_tracks, key=lambda i: i[response_name_constants.play_count], reverse=True)
play_count_sorted_saved_tracks = \
play_count_sorted_saved_tracks[slice(
offset, offset + limit, 1)]
results['saved_tracks'] = play_count_sorted_saved_tracks
if (searchKind in [SearchKind.all, SearchKind.users]):
# Query followed users that have referenced this tag
user_ids = [user['user_id'] for user in results['users']]
followed_user_query = (session.query(
Follow.followee_user_id).filter(
Follow.is_current == True, Follow.is_delete == False,
Follow.follower_user_id == current_user_id,
Follow.followee_user_id.in_(user_ids)).all())
followed_user_ids = [i[0] for i in followed_user_query]
followed_users = get_unpopulated_users(session, followed_user_ids)
followed_users = \
populate_user_metadata(
session,
followed_user_ids,
followed_users,
current_user_id
)
followed_users_followee_sorted = \
sorted(
followed_users,
key=lambda i: i[response_name_constants.follower_count],
reverse=True)
followed_users_followee_sorted = \
followed_users_followee_sorted[slice(
offset, offset + limit, 1)]
results['followed_users'] = followed_users_followee_sorted
return api_helpers.success_response(results)