def trending(time):
(limit, offset) = get_pagination_vars()
# Increment total trending count
REDIS.incr(trending_cache_total_key, 1)
genre = request.args.get("genre", default=None, type=str)
if genre is None:
redis_key = f"trending-{time}"
redis_cache_value = REDIS.get(redis_key)
if redis_cache_value is not None:
json_cache = json.loads(redis_cache_value.decode('utf-8'))
if json_cache is not None:
num_cached_entries = len(json_cache['listen_counts'])
logger.info(
f'Cache for {redis_key}, {num_cached_entries} entries, request limit {limit}'
)
if offset + limit <= num_cached_entries:
json_cache['listen_counts'] = json_cache['listen_counts'][
offset:offset + limit]
logger.info(f'Returning cache for {redis_key}')
# Increment cache hit count
REDIS.incr(trending_cache_hits_key, 1)
return api_helpers.success_response(json_cache)
# Increment cache miss count
REDIS.incr(trending_cache_miss_key, 1)
# Recalculate trending values if necessary
final_resp = generate_trending(get_db_read_replica(), time, genre, limit,
offset)
return api_helpers.success_response(final_resp)
def search(isAutocomplete):
searchStr = request.args.get("query", type=str)
if not searchStr:
return api_helpers.error_response("Invalid value for parameter 'query'")
searchStr = searchStr.replace('&', 'and') # when creating query table, we substitute this too
kind = request.args.get("kind", type=str, default="all")
if kind not in SearchKind.__members__:
return api_helpers.error_response(
"Invalid value for parameter 'kind' must be in %s" % [k.name for k in SearchKind]
)
searchKind = SearchKind[kind]
(limit, offset) = get_pagination_vars()
results = {}
if searchStr:
db = get_db_read_replica()
with db.scoped_session() as session:
# Set similarity threshold to be used by % operator in queries.
session.execute(sqlalchemy.text(f"select set_limit({minSearchSimilarity});"))
if (searchKind in [SearchKind.all, SearchKind.tracks]):
results['tracks'] = track_search_query(session, searchStr, limit, offset, False, isAutocomplete)
results['saved_tracks'] = track_search_query(session, searchStr, limit, offset, True, isAutocomplete)
if (searchKind in [SearchKind.all, SearchKind.users]):
results['users'] = user_search_query(session, searchStr, limit, offset, False, isAutocomplete)
results['followed_users'] = user_search_query(session, searchStr, limit, offset, True, isAutocomplete)
if (searchKind in [SearchKind.all, SearchKind.playlists]):
results['playlists'] = playlist_search_query(
session,
searchStr,
limit,
offset,
False,
False,
isAutocomplete
)
results['saved_playlists'] = playlist_search_query(
session,
searchStr,
limit,
offset,
False,
True,
isAutocomplete
)
if (searchKind in [SearchKind.all, SearchKind.albums]):
results['albums'] = playlist_search_query(session, searchStr, limit, offset, True, False, isAutocomplete)
results['saved_albums'] = playlist_search_query(
session,
searchStr,
limit,
offset,
True,
True,
isAutocomplete
)
return api_helpers.success_response(results)
def get_tracks_and_ids():
if "handle" in args:
handle = args.get("handle")
user_id = session.query(User.user_id).filter(
User.handle_lc == handle.lower()).first()
args["user_id"] = user_id
can_use_shared_cache = ("id" in args
and not "min_block_number" in args
and not "sort" in args
and not "user_id" in args)
if can_use_shared_cache:
should_filter_deleted = args.get("filter_deleted", False)
tracks = get_unpopulated_tracks(session, args["id"],
should_filter_deleted)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
(limit, offset) = get_pagination_vars()
args["limit"] = limit
args["offset"] = offset
tracks = _get_tracks(session, args)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
def search(isAutocomplete):
searchStr = request.args.get("query", type=str)
if not searchStr:
raise exceptions.ArgumentError("Invalid value for parameter 'query'")
searchStr = searchStr.replace('&', 'and') # when creating query table, we substitute this too
(limit, offset) = get_pagination_vars()
results = {
'tracks': [],
'users': [],
'playlists': [],
'albums': [],
'saved_tracks': [],
'followed_users': [],
'saved_playlists': [],
'saved_albums': [],
}
if searchStr:
db = get_db()
with db.scoped_session() as session:
results['tracks'] = track_search_query(session, searchStr, limit, offset, False, isAutocomplete)
results['users'] = user_search_query(session, searchStr, limit, offset, False, isAutocomplete)
results['playlists'] = playlist_search_query(
session,
searchStr,
limit,
offset,
False,
False,
isAutocomplete
)
results['albums'] = playlist_search_query(session, searchStr, limit, offset, True, False, isAutocomplete)
results['saved_tracks'] = track_search_query(session, searchStr, limit, offset, True, isAutocomplete)
results['followed_users'] = user_search_query(session, searchStr, limit, offset, True, isAutocomplete)
results['saved_playlists'] = playlist_search_query(
session,
searchStr,
limit,
offset,
False,
True,
isAutocomplete
)
results['saved_albums'] = playlist_search_query(
session,
searchStr,
limit,
offset,
True,
True,
isAutocomplete
)
return api_helpers.success_response(results)
def get_feed(args):
skip_es = request.args.get("es") == "0"
use_es = es_url and not skip_es
if use_es:
try:
(limit, _) = get_pagination_vars()
return get_feed_es(args, limit)
except:
return get_feed_sql(args)
else:
return get_feed_sql(args)
def get_followees_for_user_route(follower_user_id):
current_user_id = get_current_user_id(required=False)
(limit, offset) = get_pagination_vars()
args = {
"follower_user_id": follower_user_id,
"current_user_id": current_user_id,
"limit": limit,
"offset": offset,
}
users = get_followees_for_user(args)
return api_helpers.success_response(users)
def get_followees_for_user_route(follower_user_id):
current_user_id = get_current_user_id(required=False)
(limit, offset) = get_pagination_vars()
args = {
'follower_user_id': follower_user_id,
'current_user_id': current_user_id,
'limit': limit,
'offset': offset
}
users = get_followees_for_user(args)
return api_helpers.success_response(users)
def get_remix_track_parents_route(track_id):
args = to_dict(request.args)
if "with_users" in request.args:
args["with_users"] = parse_bool_param(request.args.get("with_users"))
args["track_id"] = track_id
args["current_user_id"] = get_current_user_id(required=False)
limit, offset = get_pagination_vars()
args["limit"] = limit
args["offset"] = offset
tracks = get_remix_track_parents(args)
return api_helpers.success_response(tracks)
def get_user_history_route(user_id):
try:
(limit, offset) = get_pagination_vars()
args = {
"user_id": user_id,
"limit": limit,
"offset": offset,
}
user_history = get_user_history(args)
return api_helpers.success_response(user_history)
except exceptions.ArgumentError as e:
return api_helpers.error_response(str(e), 400)
def get_tracks_and_ids():
if "handle" in args:
handle = args.get("handle")
user = (session.query(User.user_id).filter(
User.handle_lc == handle.lower()).first())
args["user_id"] = user.user_id
if "routes" in args:
# Convert the handles to user_ids
routes = args.get("routes")
handles = [route["handle"].lower() for route in routes]
user_id_tuples = (session.query(
User.user_id,
User.handle_lc).filter(User.handle_lc.in_(handles),
User.is_current == True).all())
user_id_map = {
handle: user_id
for (user_id, handle) in user_id_tuples
}
args["routes"] = []
for route in routes:
if route["handle"].lower() in user_id_map:
args["routes"].append({
"slug":
route["slug"],
"owner_id":
user_id_map[route["handle"].lower()],
})
# If none of the handles were found, return empty lists
if not args["routes"]:
return ([], [])
can_use_shared_cache = ("id" in args
and "min_block_number" not in args
and "sort" not in args
and "user_id" not in args)
if can_use_shared_cache:
should_filter_deleted = args.get("filter_deleted", False)
tracks = get_unpopulated_tracks(session, args["id"],
should_filter_deleted)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
(limit, offset) = get_pagination_vars()
args["limit"] = limit
args["offset"] = offset
tracks = _get_tracks(session, args)
track_ids = list(map(lambda track: track["track_id"], tracks))
return (tracks, track_ids)
def get_savers_for_playlist_route(save_playlist_id):
try:
current_user_id = get_current_user_id(required=False)
(limit, offset) = get_pagination_vars()
args = {
'save_playlist_id': save_playlist_id,
'current_user_id': current_user_id,
'limit': limit,
'offset': offset
}
user_results = get_savers_for_playlist(args)
return api_helpers.success_response(user_results)
except exceptions.NotFoundError as e:
return api_helpers.error_response(str(e), 404)
def get_savers_for_track_route(save_track_id):
try:
current_user_id = get_current_user_id(required=False)
(limit, offset) = get_pagination_vars()
args = {
"save_track_id": save_track_id,
"current_user_id": current_user_id,
"limit": limit,
"offset": offset,
}
user_results = get_savers_for_track(args)
return api_helpers.success_response(user_results)
except exceptions.NotFoundError as e:
return api_helpers.error_response(str(e), 404)
def get_remixes_of_route(track_id):
args = to_dict(request.args)
args["track_id"] = track_id
args["current_user_id"] = get_current_user_id(required=False)
limit, offset = get_pagination_vars()
args["limit"] = limit
args["offset"] = offset
if "with_users" in request.args:
args["with_users"] = parse_bool_param(request.args.get("with_users"))
try:
remixes = get_remixes_of(args)
return api_helpers.success_response(remixes)
except exceptions.ArgumentError as e:
return api_helpers.error_response(str(e), 400)
def get_trending_tracks(args):
(limit, offset) = get_pagination_vars()
current_user_id = get_current_user_id(required=False)
db = get_db_read_replica()
time = args.get('time')
# Identity understands allTime as millennium.
# TODO: Change this in https://github.com/AudiusProject/audius-protocol/pull/768/files
query_time = time
if time == 'allTime':
query_time = 'millennium'
with db.scoped_session() as session:
trending_tracks = generate_trending(get_db_read_replica(), query_time,
args.get('genre', None), limit,
offset)
track_scores = [
z(time, track) for track in trending_tracks['listen_counts']
]
sorted_track_scores = sorted(track_scores,
key=lambda k: k['score'],
reverse=True)
track_ids = [track['track_id'] for track in sorted_track_scores]
tracks = session.query(Track).filter(
Track.is_current == True, Track.is_unlisted == False,
Track.stem_of == None, Track.track_id.in_(track_ids)).all()
tracks = helpers.query_result_to_list(tracks)
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return sorted_tracks
def get_playlist_tracks(args):
playlists = []
current_user_id = args.get("current_user_id")
db = get_db_read_replica()
with db.scoped_session() as session:
try:
playlist_id = args.get("playlist_id")
playlist = (session.query(Playlist).filter(
Playlist.is_current == True,
Playlist.playlist_id == playlist_id).first())
if playlist is None:
return None
playlist_track_ids = [
track_id['track']
for track_id in playlist.playlist_contents['track_ids']
]
(limit, offset) = get_pagination_vars()
query_playlist_track_ids = playlist_track_ids[offset:offset +
limit]
playlist_tracks = (session.query(Track).filter(
Track.is_current == True,
Track.track_id.in_(query_playlist_track_ids)).all())
tracks = helpers.query_result_to_list(playlist_tracks)
tracks = populate_track_metadata(session, playlist_track_ids,
tracks, current_user_id)
if args.get("with_users", False):
add_users_to_tracks(session, tracks)
tracks_dict = {track['track_id']: track for track in tracks}
playlist_tracks = []
for track_id in query_playlist_track_ids:
playlist_tracks.append(tracks_dict[track_id])
return playlist_tracks
except sqlalchemy.orm.exc.NoResultFound:
pass
return playlists
def search_autocomplete():
args = to_dict(request.args)
validation_error = validate_search_args(args)
if validation_error:
return validation_error
current_user_id = get_current_user_id(required=False)
limit, offset = get_pagination_vars()
search_args = {
"is_auto_complete": True,
"kind": args.get("kind", "all"),
"query": args.get("query"),
"current_user_id": current_user_id,
"with_users": False,
"limit": limit,
"offset": offset
}
resp = search(search_args)
return api_helpers.success_response(resp)
def get_trending_tracks(args):
(limit, offset) = get_pagination_vars()
current_user_id = get_current_user_id(required=False)
db = get_db_read_replica()
time = args.get('time')
with db.scoped_session() as session:
trending_tracks = generate_trending(get_db_read_replica(), time,
args.get('genre', None), limit,
offset)
track_scores = [
z(time, track) for track in trending_tracks['listen_counts']
]
sorted_track_scores = sorted(track_scores,
key=lambda k: k['score'],
reverse=True)
track_ids = [track['track_id'] for track in sorted_track_scores]
tracks = session.query(Track).filter(
Track.is_current == True, Track.is_unlisted == False,
Track.stem_of == None, Track.track_id.in_(track_ids)).all()
tracks = helpers.query_result_to_list(tracks)
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
tracks_map = {track['track_id']: track for track in tracks}
# Re-sort the populated tracks b/c it loses sort order in sql query
sorted_tracks = [tracks_map[track_id] for track_id in track_ids]
if args.get("with_users", False):
user_id_list = get_users_ids(sorted_tracks)
users = get_users_by_id(session, user_id_list)
for track in sorted_tracks:
user = users[track['owner_id']]
if user:
track['user'] = user
return sorted_tracks
def search_tags():
search_str = request.args.get("query", type=str)
current_user_id = get_current_user_id(required=False)
if not search_str:
raise exceptions.ArgumentError("Invalid value for parameter 'query'")
user_tag_count = request.args.get("user_tag_count", type=str)
if not user_tag_count:
user_tag_count = "2"
kind = request.args.get("kind", type=str, default="all")
validSearchKinds = [SearchKind.all, SearchKind.tracks, SearchKind.users]
try:
searchKind = SearchKind[kind]
if searchKind not in validSearchKinds:
raise Exception
except Exception:
return api_helpers.error_response(
"Invalid value for parameter 'kind' must be in %s" %
[k.name for k in validSearchKinds], 400)
results = {}
(limit, offset) = get_pagination_vars()
like_tags_str = str.format('%{}%', search_str)
db = get_db_read_replica()
with db.scoped_session() as session:
if (searchKind in [SearchKind.all, SearchKind.tracks]):
track_res = sqlalchemy.text(f"""
select distinct(track_id)
from
(
select
strip(to_tsvector(tracks.tags)) as tagstrip,
track_id
from
tracks
where
(tags like :like_tags_query)
and (is_current is true)
and (is_delete is false)
and (is_unlisted is false)
and (stem_of is NULL)
order by
updated_at desc
) as t
where
tagstrip @@ to_tsquery(:query);
""")
track_ids = session.execute(track_res, {
"query": search_str,
"like_tags_query": like_tags_str
}).fetchall()
# track_ids is list of tuples - simplify to 1-D list
track_ids = [i[0] for i in track_ids]
tracks = (session.query(Track).filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
Track.track_id.in_(track_ids),
).all())
tracks = helpers.query_result_to_list(tracks)
track_play_counts = get_track_play_counts(track_ids)
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
for track in tracks:
track_id = track["track_id"]
track[response_name_constants.
play_count] = track_play_counts.get(track_id, 0)
play_count_sorted_tracks = \
sorted(
tracks, key=lambda i: i[response_name_constants.play_count], reverse=True)
# Add pagination parameters to track and user results
play_count_sorted_tracks = \
play_count_sorted_tracks[slice(offset, offset + limit, 1)]
results['tracks'] = play_count_sorted_tracks
if (searchKind in [SearchKind.all, SearchKind.users]):
user_res = sqlalchemy.text(f"""
select * from
(
select
count(track_id),
owner_id
from
(
select
strip(to_tsvector(tracks.tags)) as tagstrip,
track_id,
owner_id
from
tracks
where
(tags like :like_tags_query)
and (is_current is true)
and (is_unlisted is false)
and (stem_of is NULL)
order by
updated_at desc
) as t
where
tagstrip @@ to_tsquery(:query)
group by
owner_id
order by
count desc
) as usr
where
usr.count >= :user_tag_count;
""")
user_ids = session.execute(
user_res, {
"query": search_str,
"like_tags_query": like_tags_str,
"user_tag_count": user_tag_count
}).fetchall()
# user_ids is list of tuples - simplify to 1-D list
user_ids = [i[1] for i in user_ids]
users = (session.query(User).filter(
User.is_current == True, User.user_id.in_(user_ids)).all())
users = helpers.query_result_to_list(users)
users = populate_user_metadata(session, user_ids, users,
current_user_id)
followee_sorted_users = \
sorted(
users, key=lambda i: i[response_name_constants.follower_count], reverse=True)
followee_sorted_users = \
followee_sorted_users[slice(offset, offset + limit, 1)]
results['users'] = followee_sorted_users
# Add personalized results for a given user
if current_user_id:
if (searchKind in [SearchKind.all, SearchKind.tracks]):
# Query saved tracks for the current user that contain this tag
saves_query = (session.query(Save.save_item_id).filter(
Save.is_current == True, Save.is_delete == False,
Save.save_type == SaveType.track,
Save.user_id == current_user_id,
Save.save_item_id.in_(track_ids)).all())
saved_track_ids = [i[0] for i in saves_query]
saved_tracks = (session.query(Track).filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
Track.track_id.in_(saved_track_ids),
).all())
saved_tracks = helpers.query_result_to_list(saved_tracks)
for saved_track in saved_tracks:
saved_track_id = saved_track["track_id"]
saved_track[response_name_constants.play_count] = \
track_play_counts.get(saved_track_id, 0)
saved_tracks = \
populate_track_metadata(
session, saved_track_ids, saved_tracks, current_user_id)
# Sort and paginate
play_count_sorted_saved_tracks = \
sorted(
saved_tracks, key=lambda i: i[response_name_constants.play_count], reverse=True)
play_count_sorted_saved_tracks = \
play_count_sorted_saved_tracks[slice(
offset, offset + limit, 1)]
results['saved_tracks'] = play_count_sorted_saved_tracks
if (searchKind in [SearchKind.all, SearchKind.users]):
# Query followed users that have referenced this tag
followed_user_query = (session.query(
Follow.followee_user_id).filter(
Follow.is_current == True, Follow.is_delete == False,
Follow.follower_user_id == current_user_id,
Follow.followee_user_id.in_(user_ids)).all())
followed_user_ids = [i[0] for i in followed_user_query]
followed_users = (session.query(User).filter(
User.is_current == True,
User.user_id.in_(followed_user_ids)).all())
followed_users = helpers.query_result_to_list(followed_users)
followed_users = \
populate_user_metadata(
session,
followed_user_ids,
followed_users,
current_user_id
)
followed_users_followee_sorted = \
sorted(
followed_users,
key=lambda i: i[response_name_constants.follower_count],
reverse=True)
followed_users_followee_sorted = \
followed_users_followee_sorted[slice(
offset, offset + limit, 1)]
results['followed_users'] = followed_users_followee_sorted
return api_helpers.success_response(results)
def trending(time):
identity_url = shared_config['discprov']['identity_service_url']
identity_trending_endpoint = urljoin(identity_url,
f"/tracks/trending/{time}")
(limit, offset) = get_pagination_vars()
queryparams = {}
queryparams["limit"] = limit
queryparams["offset"] = offset
resp = None
try:
resp = requests.get(identity_trending_endpoint, params=queryparams)
except Exception as e:
logger.error(
f'Error retrieving trending info - {identity_trending_endpoint}, {queryparams}'
)
raise e
json_resp = resp.json()
if "error" in json_resp:
return api_helpers.error_response(json_resp["error"], 500)
listen_counts = json_resp["listenCounts"]
# Convert trackId to snakeCase
for track_entry in listen_counts:
track_entry[response_name_constants.track_id] = track_entry['trackId']
del track_entry['trackId']
track_ids = [
track[response_name_constants.track_id] for track in listen_counts
]
db = get_db()
with db.scoped_session() as session:
# Filter tracks to not-deleted ones so trending order is preserved
not_deleted_track_ids = (session.query(Track.track_id).filter(
Track.track_id.in_(track_ids), Track.is_current == True,
Track.is_delete == False).all())
not_deleted_track_ids = set(
[record[0] for record in not_deleted_track_ids])
# Query repost counts
repost_counts = get_repost_counts(session, False, True,
not_deleted_track_ids, None)
track_repost_counts = {
repost_item_id: repost_count
for (repost_item_id, repost_count, repost_type) in repost_counts
if repost_type == RepostType.track
}
# Query follower info for each track owner
# Query each track owner
track_owners_query = (session.query(
Track.track_id, Track.owner_id).filter(
Track.is_current == True,
Track.track_id.in_(not_deleted_track_ids)).all())
# Generate track_id <-> owner_id mapping
track_owner_dict = {
track_id: owner_id
for (track_id, owner_id) in track_owners_query
}
# Generate list of owner ids
track_owner_list = [
owner_id for (track_id, owner_id) in track_owners_query
]
# build dict of owner_id --> follower_count
follower_counts = (session.query(
Follow.followee_user_id,
func.count(Follow.followee_user_id)).filter(
Follow.is_current == True, Follow.is_delete == False,
Follow.followee_user_id.in_(track_owner_list)).group_by(
Follow.followee_user_id).all())
follower_count_dict = \
{user_id: follower_count for (user_id, follower_count) in follower_counts}
save_counts = get_save_counts(session, False, True,
not_deleted_track_ids, None)
track_save_counts = {
save_item_id: save_count
for (save_item_id, save_count, save_type) in save_counts
if save_type == SaveType.track
}
trending = []
for track_entry in listen_counts:
# Skip over deleted tracks
if (track_entry[response_name_constants.track_id]
not in not_deleted_track_ids):
continue
# Populate repost counts
if track_entry[
response_name_constants.track_id] in track_repost_counts:
track_entry[response_name_constants.repost_count] = \
track_repost_counts[track_entry[response_name_constants.track_id]]
else:
track_entry[response_name_constants.repost_count] = 0
# Populate save counts
if track_entry[
response_name_constants.track_id] in track_save_counts:
track_entry[response_name_constants.save_count] = \
track_save_counts[track_entry[response_name_constants.track_id]]
else:
track_entry[response_name_constants.save_count] = 0
# Populate listen counts
owner_id = track_owner_dict[track_entry[
response_name_constants.track_id]]
owner_follow_count = 0
if owner_id in follower_count_dict:
owner_follow_count = follower_count_dict[owner_id]
track_entry[response_name_constants.track_owner_id] = owner_id
track_entry[response_name_constants.
track_owner_follower_count] = owner_follow_count
trending.append(track_entry)
final_resp = {}
final_resp['listen_counts'] = trending
return api_helpers.success_response(final_resp)
def get_feed_sql(args):
feed_results = []
db = get_db_read_replica()
feed_filter = args.get("filter")
# Allow for fetching only tracks
tracks_only = args.get("tracks_only", False)
followee_user_ids = args.get("followee_user_ids", [])
# Current user - user for whom feed is being generated
current_user_id = args.get("user_id")
with db.scoped_session() as session:
# Generate list of users followed by current user, i.e. 'followees'
if not followee_user_ids:
followee_user_ids = (
session.query(Follow.followee_user_id)
.filter(
Follow.follower_user_id == current_user_id,
Follow.is_current == True,
Follow.is_delete == False,
)
.all()
)
followee_user_ids = [f[0] for f in followee_user_ids]
# Fetch followee creations if requested
if feed_filter in ["original", "all"]:
if not tracks_only:
# Query playlists posted by followees, sorted and paginated by created_at desc
created_playlists_query = (
session.query(Playlist)
.filter(
Playlist.is_current == True,
Playlist.is_delete == False,
Playlist.is_private == False,
Playlist.playlist_owner_id.in_(followee_user_ids),
)
.order_by(desc(Playlist.created_at))
)
created_playlists = paginate_query(created_playlists_query, False).all()
# get track ids for all tracks in playlists
playlist_track_ids = set()
for playlist in created_playlists:
for track in playlist.playlist_contents["track_ids"]:
playlist_track_ids.add(track["track"])
# get all track objects for track ids
playlist_tracks = get_unpopulated_tracks(session, playlist_track_ids)
playlist_tracks_dict = {
track["track_id"]: track for track in playlist_tracks
}
# get all track ids that have same owner as playlist and created in "same action"
# "same action": track created within [x time] before playlist creation
tracks_to_dedupe = set()
for playlist in created_playlists:
for track_entry in playlist.playlist_contents["track_ids"]:
track = playlist_tracks_dict.get(track_entry["track"])
if not track:
continue
max_timedelta = datetime.timedelta(
minutes=trackDedupeMaxMinutes
)
if (
(track["owner_id"] == playlist.playlist_owner_id)
and (track["created_at"] <= playlist.created_at)
and (
playlist.created_at - track["created_at"]
<= max_timedelta
)
):
tracks_to_dedupe.add(track["track_id"])
tracks_to_dedupe = list(tracks_to_dedupe)
else:
# No playlists to consider
tracks_to_dedupe = []
created_playlists = []
# Query tracks posted by followees, sorted & paginated by created_at desc
# exclude tracks that were posted in "same action" as playlist
created_tracks_query = (
session.query(Track)
.filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
Track.owner_id.in_(followee_user_ids),
Track.track_id.notin_(tracks_to_dedupe),
)
.order_by(desc(Track.created_at))
)
created_tracks = paginate_query(created_tracks_query, False).all()
# extract created_track_ids and created_playlist_ids
created_track_ids = [track.track_id for track in created_tracks]
created_playlist_ids = [
playlist.playlist_id for playlist in created_playlists
]
# Fetch followee reposts if requested
if feed_filter in ["repost", "all"]:
# query items reposted by followees, sorted by oldest followee repost of item;
# paginated by most recent repost timestamp
repost_subquery = session.query(Repost).filter(
Repost.is_current == True,
Repost.is_delete == False,
Repost.user_id.in_(followee_user_ids),
)
# exclude items also created by followees to guarantee order determinism, in case of "all" filter
if feed_filter == "all":
repost_subquery = repost_subquery.filter(
or_(
and_(
Repost.repost_type == RepostType.track,
Repost.repost_item_id.notin_(created_track_ids),
),
and_(
Repost.repost_type != RepostType.track,
Repost.repost_item_id.notin_(created_playlist_ids),
),
)
)
repost_subquery = repost_subquery.subquery()
repost_query = (
session.query(
repost_subquery.c.repost_item_id,
repost_subquery.c.repost_type,
func.min(repost_subquery.c.created_at).label("min_created_at"),
)
.group_by(
repost_subquery.c.repost_item_id, repost_subquery.c.repost_type
)
.order_by(desc("min_created_at"))
)
followee_reposts = paginate_query(repost_query, False).all()
# build dict of track_id / playlist_id -> oldest followee repost timestamp from followee_reposts above
track_repost_timestamp_dict = {}
playlist_repost_timestamp_dict = {}
for (
repost_item_id,
repost_type,
oldest_followee_repost_timestamp,
) in followee_reposts:
if repost_type == RepostType.track:
track_repost_timestamp_dict[
repost_item_id
] = oldest_followee_repost_timestamp
elif repost_type in (RepostType.playlist, RepostType.album):
playlist_repost_timestamp_dict[
repost_item_id
] = oldest_followee_repost_timestamp
# extract reposted_track_ids and reposted_playlist_ids
reposted_track_ids = list(track_repost_timestamp_dict.keys())
reposted_playlist_ids = list(playlist_repost_timestamp_dict.keys())
# Query tracks reposted by followees
reposted_tracks = session.query(Track).filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
Track.track_id.in_(reposted_track_ids),
)
# exclude tracks already fetched from above, in case of "all" filter
if feed_filter == "all":
reposted_tracks = reposted_tracks.filter(
Track.track_id.notin_(created_track_ids)
)
reposted_tracks = reposted_tracks.order_by(desc(Track.created_at)).all()
if not tracks_only:
# Query playlists reposted by followees, excluding playlists already fetched from above
reposted_playlists = session.query(Playlist).filter(
Playlist.is_current == True,
Playlist.is_delete == False,
Playlist.is_private == False,
Playlist.playlist_id.in_(reposted_playlist_ids),
)
# exclude playlists already fetched from above, in case of "all" filter
if feed_filter == "all":
reposted_playlists = reposted_playlists.filter(
Playlist.playlist_id.notin_(created_playlist_ids)
)
reposted_playlists = reposted_playlists.order_by(
desc(Playlist.created_at)
).all()
else:
reposted_playlists = []
if feed_filter == "original":
tracks_to_process = created_tracks
playlists_to_process = created_playlists
elif feed_filter == "repost":
tracks_to_process = reposted_tracks
playlists_to_process = reposted_playlists
else:
tracks_to_process = created_tracks + reposted_tracks
playlists_to_process = created_playlists + reposted_playlists
tracks = helpers.query_result_to_list(tracks_to_process)
playlists = helpers.query_result_to_list(playlists_to_process)
# define top level feed activity_timestamp to enable sorting
# activity_timestamp: created_at if item created by followee, else reposted_at
for track in tracks:
if track["owner_id"] in followee_user_ids:
track[response_name_constants.activity_timestamp] = track["created_at"]
else:
track[
response_name_constants.activity_timestamp
] = track_repost_timestamp_dict[track["track_id"]]
for playlist in playlists:
if playlist["playlist_owner_id"] in followee_user_ids:
playlist[response_name_constants.activity_timestamp] = playlist[
"created_at"
]
else:
playlist[
response_name_constants.activity_timestamp
] = playlist_repost_timestamp_dict[playlist["playlist_id"]]
# bundle peripheral info into track and playlist objects
track_ids = list(map(lambda track: track["track_id"], tracks))
playlist_ids = list(map(lambda playlist: playlist["playlist_id"], playlists))
tracks = populate_track_metadata(session, track_ids, tracks, current_user_id)
playlists = populate_playlist_metadata(
session,
playlist_ids,
playlists,
[RepostType.playlist, RepostType.album],
[SaveType.playlist, SaveType.album],
current_user_id,
)
# build combined feed of tracks and playlists
unsorted_feed = tracks + playlists
# sort feed based on activity_timestamp
sorted_feed = sorted(
unsorted_feed,
key=lambda entry: entry[response_name_constants.activity_timestamp],
reverse=True,
)
# truncate feed to requested limit
(limit, _) = get_pagination_vars()
feed_results = sorted_feed[0:limit]
if "with_users" in args and args.get("with_users") != False:
user_id_list = get_users_ids(feed_results)
users = get_users_by_id(session, user_id_list)
for result in feed_results:
if "playlist_owner_id" in result:
user = users[result["playlist_owner_id"]]
if user:
result["user"] = user
elif "owner_id" in result:
user = users[result["owner_id"]]
if user:
result["user"] = user
return feed_results
def search_tags():
search_str = request.args.get("query", type=str)
current_user_id = get_current_user_id(required=False)
if not search_str:
raise exceptions.ArgumentError("Invalid value for parameter 'query'")
user_tag_count = request.args.get("user_tag_count", type=str)
if not user_tag_count:
user_tag_count = "2"
kind = request.args.get("kind", type=str, default="all")
validSearchKinds = [SearchKind.all, SearchKind.tracks, SearchKind.users]
try:
searchKind = SearchKind[kind]
if searchKind not in validSearchKinds:
raise Exception
except Exception:
return api_helpers.error_response(
"Invalid value for parameter 'kind' must be in %s" %
[k.name for k in validSearchKinds], 400)
results = {}
(limit, offset) = get_pagination_vars()
db = get_db_read_replica()
with db.scoped_session() as session:
if (searchKind in [SearchKind.all, SearchKind.tracks]):
results['tracks'] = search_track_tags(
session, {
'search_str': search_str,
'current_user_id': current_user_id,
'limit': limit,
'offset': offset
})
if (searchKind in [SearchKind.all, SearchKind.users]):
results['users'] = search_user_tags(
session, {
'search_str': search_str,
'current_user_id': current_user_id,
"user_tag_count": user_tag_count,
'limit': limit,
'offset': offset
})
# Add personalized results for a given user
if current_user_id:
if (searchKind in [SearchKind.all, SearchKind.tracks]):
# Query saved tracks for the current user that contain this tag
track_ids = [track['track_id'] for track in results['tracks']]
track_play_counts = {
track['track_id']: track[response_name_constants.play_count]
for track in results['tracks']
}
saves_query = (session.query(Save.save_item_id).filter(
Save.is_current == True, Save.is_delete == False,
Save.save_type == SaveType.track,
Save.user_id == current_user_id,
Save.save_item_id.in_(track_ids)).all())
saved_track_ids = [i[0] for i in saves_query]
saved_tracks = (session.query(Track).filter(
Track.is_current == True,
Track.is_delete == False,
Track.is_unlisted == False,
Track.stem_of == None,
Track.track_id.in_(saved_track_ids),
).all())
saved_tracks = helpers.query_result_to_list(saved_tracks)
for saved_track in saved_tracks:
saved_track_id = saved_track["track_id"]
saved_track[response_name_constants.play_count] = \
track_play_counts.get(saved_track_id, 0)
saved_tracks = \
populate_track_metadata(
session, saved_track_ids, saved_tracks, current_user_id)
# Sort and paginate
play_count_sorted_saved_tracks = \
sorted(
saved_tracks, key=lambda i: i[response_name_constants.play_count], reverse=True)
play_count_sorted_saved_tracks = \
play_count_sorted_saved_tracks[slice(
offset, offset + limit, 1)]
results['saved_tracks'] = play_count_sorted_saved_tracks
if (searchKind in [SearchKind.all, SearchKind.users]):
# Query followed users that have referenced this tag
user_ids = [user['user_id'] for user in results['users']]
followed_user_query = (session.query(
Follow.followee_user_id).filter(
Follow.is_current == True, Follow.is_delete == False,
Follow.follower_user_id == current_user_id,
Follow.followee_user_id.in_(user_ids)).all())
followed_user_ids = [i[0] for i in followed_user_query]
followed_users = get_unpopulated_users(session, followed_user_ids)
followed_users = \
populate_user_metadata(
session,
followed_user_ids,
followed_users,
current_user_id
)
followed_users_followee_sorted = \
sorted(
followed_users,
key=lambda i: i[response_name_constants.follower_count],
reverse=True)
followed_users_followee_sorted = \
followed_users_followee_sorted[slice(
offset, offset + limit, 1)]
results['followed_users'] = followed_users_followee_sorted
return api_helpers.success_response(results)
def get_feed():
feed_results = []
db = get_db()
# Current user - user for whom feed is being generated
current_user_id = get_current_user_id()
with db.scoped_session() as session:
# Generate list of users followed by current user, i.e. 'followees'
followee_user_ids = (session.query(Follow.followee_user_id).filter(
Follow.follower_user_id == current_user_id,
Follow.is_current == True, Follow.is_delete == False).all())
followee_user_ids = [f[0] for f in followee_user_ids]
# Query playlists posted by followees, sorted and paginated by created_at desc
created_playlists_query = (session.query(Playlist).filter(
Playlist.is_current == True, Playlist.is_private == False,
Playlist.playlist_owner_id.in_(followee_user_ids)).order_by(
desc(Playlist.created_at)))
created_playlists = paginate_query(created_playlists_query,
False).all()
# get track ids for all tracks in playlists
playlist_track_ids = set()
for playlist in created_playlists:
for track in playlist.playlist_contents["track_ids"]:
playlist_track_ids.add(track["track"])
# get all track objects for track ids
playlist_tracks = (session.query(Track).filter(
Track.is_current == True,
Track.track_id.in_(playlist_track_ids)).all())
playlist_tracks_dict = {
track.track_id: track
for track in playlist_tracks
}
# get all track ids that have same owner as playlist and created in "same action"
# "same action": track created within [x time] before playlist creation
tracks_to_dedupe = set()
for playlist in created_playlists:
for track_entry in playlist.playlist_contents["track_ids"]:
track = playlist_tracks_dict.get(track_entry["track"])
if not track:
return api_helpers.error_response(
"Something caused the server to crash.")
max_timedelta = datetime.timedelta(
minutes=trackDedupeMaxMinutes)
if (track.owner_id == playlist.playlist_owner_id) and \
(track.created_at <= playlist.created_at) and \
(playlist.created_at - track.created_at <= max_timedelta):
tracks_to_dedupe.add(track.track_id)
tracks_to_dedupe = list(tracks_to_dedupe)
# Query tracks posted by followees, sorted & paginated by created_at desc
# exclude tracks that were posted in "same action" as playlist
created_tracks_query = (session.query(Track).filter(
Track.is_current == True, Track.owner_id.in_(followee_user_ids),
Track.track_id.notin_(tracks_to_dedupe)).order_by(
desc(Track.created_at)))
created_tracks = paginate_query(created_tracks_query, False).all()
# extract created_track_ids and created_playlist_ids
created_track_ids = [track.track_id for track in created_tracks]
created_playlist_ids = [
playlist.playlist_id for playlist in created_playlists
]
# query items reposted by followees, sorted by oldest followee repost of item;
# paginated by most recent repost timestamp
# exclude items also created by followees to guarantee order determinism
repost_subquery = (session.query(Repost).filter(
Repost.is_current == True, Repost.is_delete == False,
Repost.user_id.in_(followee_user_ids),
or_(
and_(Repost.repost_type == RepostType.track,
Repost.repost_item_id.notin_(created_track_ids)),
and_(Repost.repost_type == RepostType.track,
Repost.repost_item_id.notin_(
created_playlist_ids)))).subquery())
repost_query = (session.query(
repost_subquery.c.repost_item_id, repost_subquery.c.repost_type,
func.min(repost_subquery.c.created_at).label("min_created_at")
).group_by(
repost_subquery.c.repost_item_id,
repost_subquery.c.repost_type).order_by("min_created_at desc"))
followee_reposts = paginate_query(repost_query, False).all()
# build dict of track id -> oldest followee repost timestamp from followee_reposts above
track_repost_timestamp_dict = {}
playlist_repost_timestamp_dict = {}
for (repost_item_id, repost_type,
oldest_followee_repost_timestamp) in followee_reposts:
if repost_type == RepostType.track:
track_repost_timestamp_dict[
repost_item_id] = oldest_followee_repost_timestamp
elif repost_type in (RepostType.playlist, RepostType.album):
playlist_repost_timestamp_dict[
repost_item_id] = oldest_followee_repost_timestamp
# extract reposted_track_ids and reposted_playlist_ids
reposted_track_ids = list(track_repost_timestamp_dict.keys())
reposted_playlist_ids = list(playlist_repost_timestamp_dict.keys())
# Query tracks reposted by followees, excluding tracks already fetched from above
reposted_tracks = (session.query(Track).filter(
Track.is_current == True, Track.track_id.in_(reposted_track_ids),
Track.track_id.notin_(created_track_ids)).order_by(
desc(Track.created_at)).all())
# Query playlists reposted by followees, excluding playlists already fetched from above
reposted_playlists = (session.query(Playlist).filter(
Playlist.is_current == True, Playlist.is_private == False,
Playlist.playlist_id.in_(reposted_playlist_ids),
Playlist.playlist_id.notin_(created_playlist_ids)).all())
# Combine created + reposted track and playlist lists
tracks = helpers.query_result_to_list(created_tracks + reposted_tracks)
playlists = helpers.query_result_to_list(created_playlists +
reposted_playlists)
# define top level feed activity_timestamp to enable sorting
# activity_timestamp: created_at if item created by followee, else reposted_at
for track in tracks:
if track["owner_id"] in followee_user_ids:
track[response_name_constants.
activity_timestamp] = track["created_at"]
else:
track[response_name_constants.
activity_timestamp] = track_repost_timestamp_dict[
track["track_id"]]
for playlist in playlists:
if playlist["playlist_owner_id"] in followee_user_ids:
playlist[response_name_constants.
activity_timestamp] = playlist["created_at"]
else:
playlist[response_name_constants.activity_timestamp] = \
playlist_repost_timestamp_dict[playlist["playlist_id"]]
# bundle peripheral info into track and playlist objects
track_ids = list(map(lambda track: track["track_id"], tracks))
playlist_ids = list(
map(lambda playlist: playlist["playlist_id"], playlists))
tracks = populate_track_metadata(session, track_ids, tracks,
current_user_id)
playlists = populate_playlist_metadata(
session, playlist_ids, playlists,
[RepostType.playlist, RepostType.album],
[SaveType.playlist, SaveType.album], current_user_id)
# build combined feed of tracks and playlists
unsorted_feed = tracks + playlists
# sort feed based on activity_timestamp
sorted_feed = sorted(unsorted_feed,
key=lambda entry: entry[response_name_constants.
activity_timestamp],
reverse=True)
# truncate feed to requested limit
(limit, _) = get_pagination_vars()
feed_results = sorted_feed[0:limit]
return api_helpers.success_response(feed_results)
def search_tags():
search_str = request.args.get("query", type=str)
current_user_id = get_current_user_id(required=False)
if not search_str:
raise exceptions.ArgumentError("Invalid value for parameter 'query'")
user_tag_count = request.args.get("user_tag_count", type=str)
if not user_tag_count:
user_tag_count = "2"
kind = request.args.get("kind", type=str, default="all")
validSearchKinds = [SearchKind.all, SearchKind.tracks, SearchKind.users]
try:
searchKind = SearchKind[kind]
if searchKind not in validSearchKinds:
raise Exception
except Exception:
return api_helpers.error_response(
f"Invalid value for parameter 'kind' must be in {[k.name for k in validSearchKinds]}",
400,
)
results = {}
(limit, offset) = get_pagination_vars()
db = get_db_read_replica()
with db.scoped_session() as session:
if searchKind in [SearchKind.all, SearchKind.tracks]:
results["tracks"] = search_track_tags(
session,
{
"search_str": search_str,
"current_user_id": current_user_id,
"limit": limit,
"offset": offset,
},
)
if searchKind in [SearchKind.all, SearchKind.users]:
results["users"] = search_user_tags(
session,
{
"search_str": search_str,
"current_user_id": current_user_id,
"user_tag_count": user_tag_count,
"limit": limit,
"offset": offset,
},
)
# Add personalized results for a given user
if current_user_id:
if searchKind in [SearchKind.all, SearchKind.tracks]:
# Query saved tracks for the current user that contain this tag
track_ids = [track["track_id"] for track in results["tracks"]]
saves_query = (
session.query(Save.save_item_id)
.filter(
Save.is_current == True,
Save.is_delete == False,
Save.save_type == SaveType.track,
Save.user_id == current_user_id,
Save.save_item_id.in_(track_ids),
)
.all()
)
saved_track_ids = {i[0] for i in saves_query}
saved_tracks = list(
filter(
lambda track: track["track_id"] in saved_track_ids,
results["tracks"],
)
)
results["saved_tracks"] = saved_tracks
if searchKind in [SearchKind.all, SearchKind.users]:
# Query followed users that have referenced this tag
user_ids = [user["user_id"] for user in results["users"]]
followed_user_query = (
session.query(Follow.followee_user_id)
.filter(
Follow.is_current == True,
Follow.is_delete == False,
Follow.follower_user_id == current_user_id,
Follow.followee_user_id.in_(user_ids),
)
.all()
)
followed_user_ids = {i[0] for i in followed_user_query}
followed_users = list(
filter(
lambda user: user["user_id"] in followed_user_ids, results["users"]
)
)
results["followed_users"] = followed_users
return api_helpers.success_response(results)
def get_top_users(current_user_id):
"""Gets the top users by follows of all of Audius"""
(limit, offset) = get_pagination_vars()
db = get_db_read_replica()
with db.scoped_session() as session:
return _get_top_users(session, current_user_id, limit, offset)