def _get_underground_trending(args, strategy): db = get_db_read_replica() with db.scoped_session() as session: current_user_id = args.get("current_user_id", None) limit, offset = args.get("limit"), args.get("offset") key = make_underground_trending_cache_key(strategy.version) (tracks, track_ids) = use_redis_cache( key, None, make_get_unpopulated_tracks(session, redis, strategy)) # Apply limit + offset early to reduce the amount of # population work we have to do if limit is not None and offset is not None: track_ids = track_ids[offset:limit + offset] tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) tracks_map = {track['track_id']: track for track in tracks} # Re-sort the populated tracks b/c it loses sort order in sql query sorted_tracks = [tracks_map[track_id] for track_id in track_ids] user_id_list = get_users_ids(sorted_tracks) users = get_users_by_id(session, user_id_list, current_user_id) for track in sorted_tracks: user = users[track['owner_id']] if user: track['user'] = user sorted_tracks = list(map(extend_track, sorted_tracks)) return sorted_tracks
def get_trending_tracks(args, strategy): """Gets trending by getting the currently cached tracks and then populating them.""" db = get_db_read_replica() with db.scoped_session() as session: current_user_id, genre, time = args.get("current_user_id"), args.get( "genre"), args.get("time", "week") time_range = "week" if time not in ["week", "month", "year"] else time key = make_trending_cache_key(time_range, genre, strategy.version) # Will try to hit cached trending from task, falling back # to generating it here if necessary and storing it with no TTL (tracks, track_ids) = use_redis_cache( key, None, make_generate_unpopulated_trending(session, genre, time_range, strategy)) # populate track metadata tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) tracks_map = {track['track_id']: track for track in tracks} # Re-sort the populated tracks b/c it loses sort order in sql query sorted_tracks = [tracks_map[track_id] for track_id in track_ids] if args.get("with_users", False): user_id_list = get_users_ids(sorted_tracks) users = get_users_by_id(session, user_id_list, current_user_id) for track in sorted_tracks: user = users[track['owner_id']] if user: track['user'] = user return sorted_tracks
def _get_underground_trending_with_session( session: Session, args: GetUndergroundTrendingTrackcArgs, strategy, use_request_context=True, ): current_user_id = args.get("current_user_id", None) limit, offset = args.get("limit"), args.get("offset") key = make_underground_trending_cache_key(strategy.version) (tracks, track_ids) = use_redis_cache( key, None, make_get_unpopulated_tracks(session, redis_conn, strategy) ) # Apply limit + offset early to reduce the amount of # population work we have to do if limit is not None and offset is not None: track_ids = track_ids[offset : limit + offset] tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) tracks_map = {track["track_id"]: track for track in tracks} # Re-sort the populated tracks b/c it loses sort order in sql query sorted_tracks = [tracks_map[track_id] for track_id in track_ids] user_id_list = get_users_ids(sorted_tracks) users = get_users_by_id(session, user_id_list, current_user_id, use_request_context) for track in sorted_tracks: user = users[track["owner_id"]] if user: track["user"] = user sorted_tracks = list(map(extend_track, sorted_tracks)) return sorted_tracks
def get_top_followee_windowed(type, window, args): if type != "track": raise exceptions.ArgumentError( "Invalid type provided, must be one of 'track'") valid_windows = ["week", "month", "year"] if not window or window not in valid_windows: raise exceptions.ArgumentError( f"Invalid window provided, must be one of {valid_windows}") limit = args.get("limit", 25) current_user_id = args.get("user_id") db = get_db_read_replica() with db.scoped_session() as session: followee_user_ids = session.query(Follow.followee_user_id).filter( Follow.follower_user_id == current_user_id, Follow.is_current == True, Follow.is_delete == False, ) followee_user_ids_subquery = followee_user_ids.subquery() # Queries for tracks joined against followed users and counts tracks_query = ( session.query(Track, ).join( followee_user_ids_subquery, Track.owner_id == followee_user_ids_subquery.c.followee_user_id, ).join(AggregateTrack, Track.track_id == AggregateTrack.track_id). filter( Track.is_current == True, Track.is_delete == False, Track.is_unlisted == False, Track.stem_of == None, # Query only tracks created `window` time ago (week, month, etc.) Track.created_at >= text(f"NOW() - interval '1 {window}'"), ).order_by( desc(AggregateTrack.repost_count + AggregateTrack.save_count), desc(Track.track_id), ).limit(limit)) tracks_query_results = tracks_query.all() tracks = helpers.query_result_to_list(tracks_query_results) track_ids = list(map(lambda track: track["track_id"], tracks)) # Bundle peripheral info into track results tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) if args.get("with_users", False): user_id_list = get_users_ids(tracks) users = get_users_by_id(session, user_id_list) for track in tracks: user = users[track["owner_id"]] if user: track["user"] = user return tracks
def get_top_followee_saves(saveType, args): if saveType != 'track': raise exceptions.ArgumentError( "Invalid type provided, must be one of 'track'") limit = args.get('limit', 25) current_user_id = get_current_user_id() db = get_db_read_replica() with db.scoped_session() as session: # Construct a subquery of all followees followee_user_ids = (session.query(Follow.followee_user_id).filter( Follow.follower_user_id == current_user_id, Follow.is_current == True, Follow.is_delete == False)) followee_user_ids_subquery = followee_user_ids.subquery() # Construct a subquery of all saves from followees aggregated by id save_count = (session.query( Save.save_item_id, func.count(Save.save_item_id).label( response_name_constants.save_count)).join( followee_user_ids_subquery, Save.user_id == followee_user_ids_subquery.c.followee_user_id).filter( Save.is_current == True, Save.is_delete == False, Save.save_type == saveType, ).group_by(Save.save_item_id).order_by( desc(response_name_constants.save_count)).limit(limit)) save_count_subquery = save_count.subquery() # Query for tracks joined against followee save counts tracks_query = (session.query(Track, ).join( save_count_subquery, Track.track_id == save_count_subquery.c.save_item_id).filter( Track.is_current == True, Track.is_delete == False, Track.is_unlisted == False, Track.stem_of == None, )) tracks_query_results = tracks_query.all() tracks = helpers.query_result_to_list(tracks_query_results) track_ids = list(map(lambda track: track['track_id'], tracks)) # bundle peripheral info into track results tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) if args.get('with_users', False): user_id_list = get_users_ids(tracks) users = get_users_by_id(session, user_id_list) for track in tracks: user = users[track['owner_id']] if user: track['user'] = user return tracks
def get_trending_tracks(args): limit, offset, current_user_id = args.get("limit"), args.get( "offset"), args.get("current_user_id") db = get_db_read_replica() time = args.get('time') query_time = None if time not in ["day", "week", "month", "year"] else time with db.scoped_session() as session: def get_unpopulated_trending(): trending_tracks = generate_trending(session, query_time, args.get('genre', None), limit, offset) track_scores = [ z(time, track) for track in trending_tracks['listen_counts'] ] sorted_track_scores = sorted(track_scores, key=lambda k: k['score'], reverse=True) track_ids = [track['track_id'] for track in sorted_track_scores] tracks = get_unpopulated_tracks(session, track_ids) return (tracks, track_ids) # get scored trending tracks, either # through cached redis value, or through `get_unpopulated_trending` cache_keys = {"genre": args.get("genre"), "time": args.get("time")} key = extract_key(f"generated-trending:{request.path}", cache_keys.items()) (tracks, track_ids) = use_redis_cache(key, SCORES_CACHE_DURATION_SEC, get_unpopulated_trending) # populate track metadata tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) tracks_map = {track['track_id']: track for track in tracks} # Re-sort the populated tracks b/c it loses sort order in sql query sorted_tracks = [tracks_map[track_id] for track_id in track_ids] if args.get("with_users", False): user_id_list = get_users_ids(sorted_tracks) users = get_users_by_id(session, user_id_list, current_user_id) for track in sorted_tracks: user = users[track['owner_id']] if user: track['user'] = user return sorted_tracks
def add_users(session, results): user_id_list = get_users_ids(results) users = get_users_by_id(session, user_id_list) for result in results: user_id = None if 'playlist_owner_id' in result: user_id = result['playlist_owner_id'] elif 'owner_id' in result: user_id = result['owner_id'] if user_id is not None: user = users[user_id] result["user"] = user return results
def get_save_tracks(args): user_id = args.get("user_id") current_user_id = args.get("current_user_id") limit = args.get("limit") offset = args.get("offset") filter_deleted = args.get("filter_deleted") db = get_db_read_replica() with db.scoped_session() as session: base_query = (session.query(Track, Save.created_at).join( Save, Save.save_item_id == Track.track_id).filter( Track.is_current == True, Save.user_id == user_id, Save.is_current == True, Save.is_delete == False, Save.save_type == SaveType.track, )) # Allow filtering of deletes if filter_deleted: base_query = base_query.filter(Track.is_delete == False) base_query = base_query.order_by(Save.created_at.desc(), Track.track_id.desc()) query_results = add_query_pagination(base_query, limit, offset).all() if not query_results: return [] tracks, save_dates = zip(*query_results) tracks = helpers.query_result_to_list(tracks) track_ids = list(map(lambda track: track["track_id"], tracks)) # bundle peripheral info into track results tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) if args.get("with_users", False): user_id_list = get_users_ids(tracks) users = get_users_by_id(session, user_id_list, current_user_id) for track in tracks: user = users[track["owner_id"]] if user: track["user"] = user for idx, track in enumerate(tracks): track[response_name_constants.activity_timestamp] = save_dates[idx] return tracks
def get_trending_tracks(args): (limit, offset) = get_pagination_vars() current_user_id = get_current_user_id(required=False) db = get_db_read_replica() time = args.get('time') # Identity understands allTime as millennium. # TODO: Change this in https://github.com/AudiusProject/audius-protocol/pull/768/files query_time = time if time == 'allTime': query_time = 'millennium' with db.scoped_session() as session: trending_tracks = generate_trending(get_db_read_replica(), query_time, args.get('genre', None), limit, offset) track_scores = [ z(time, track) for track in trending_tracks['listen_counts'] ] sorted_track_scores = sorted(track_scores, key=lambda k: k['score'], reverse=True) track_ids = [track['track_id'] for track in sorted_track_scores] tracks = session.query(Track).filter( Track.is_current == True, Track.is_unlisted == False, Track.stem_of == None, Track.track_id.in_(track_ids)).all() tracks = helpers.query_result_to_list(tracks) tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) tracks_map = {track['track_id']: track for track in tracks} # Re-sort the populated tracks b/c it loses sort order in sql query sorted_tracks = [tracks_map[track_id] for track_id in track_ids] if args.get("with_users", False): user_id_list = get_users_ids(sorted_tracks) users = get_users_by_id(session, user_id_list) for track in sorted_tracks: user = users[track['owner_id']] if user: track['user'] = user return sorted_tracks
def get_support_sent_by_user(args) -> List[SupportResponse]: support: List[SupportResponse] = [] sender_user_id = args.get("user_id") current_user_id = args.get("current_user_id") limit = args.get("limit") offset = args.get("offset") db = get_db_read_replica() with db.scoped_session() as session: query = ( session.query("rank", AggregateUserTips) .from_statement(sql_support_sent) .params(sender_user_id=sender_user_id, limit=limit, offset=offset) ) rows: List[Tuple[int, AggregateUserTips]] = query.all() user_ids = [row[1].receiver_user_id for row in rows] users = get_users_by_id(session, user_ids, current_user_id) support = query_result_to_support_response(rows, users, user_is_sender=False) return support
def get_trending_tracks(args): (limit, offset) = get_pagination_vars() current_user_id = get_current_user_id(required=False) db = get_db_read_replica() time = args.get('time') with db.scoped_session() as session: trending_tracks = generate_trending(get_db_read_replica(), time, args.get('genre', None), limit, offset) track_scores = [ z(time, track) for track in trending_tracks['listen_counts'] ] sorted_track_scores = sorted(track_scores, key=lambda k: k['score'], reverse=True) track_ids = [track['track_id'] for track in sorted_track_scores] tracks = session.query(Track).filter( Track.is_current == True, Track.is_unlisted == False, Track.stem_of == None, Track.track_id.in_(track_ids)).all() tracks = helpers.query_result_to_list(tracks) tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) tracks_map = {track['track_id']: track for track in tracks} # Re-sort the populated tracks b/c it loses sort order in sql query sorted_tracks = [tracks_map[track_id] for track_id in track_ids] if args.get("with_users", False): user_id_list = get_users_ids(sorted_tracks) users = get_users_by_id(session, user_id_list) for track in sorted_tracks: user = users[track['owner_id']] if user: track['user'] = user return sorted_tracks
def _get_trending_tracks_with_session(session: Session, args: GetTrendingTracksArgs, strategy: BaseTrendingStrategy): current_user_id, genre, time = ( args.get("current_user_id"), args.get("genre"), args.get("time", "week"), ) time_range = "week" if time not in ["week", "month", "year", "allTime" ] else time key = make_trending_cache_key(time_range, genre, strategy.version) # Will try to hit cached trending from task, falling back # to generating it here if necessary and storing it with no TTL (tracks, track_ids) = use_redis_cache( key, None, make_generate_unpopulated_trending(session, genre, time_range, strategy), ) # populate track metadata tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) tracks_map = {track["track_id"]: track for track in tracks} # Re-sort the populated tracks b/c it loses sort order in sql query sorted_tracks = [tracks_map[track_id] for track_id in track_ids] if args.get("with_users", False): user_id_list = get_users_ids(sorted_tracks) users = get_users_by_id(session, user_id_list, current_user_id) for track in sorted_tracks: user = users[track["owner_id"]] if user: track["user"] = user return sorted_tracks
def get_playlists(args): playlists = [] current_user_id = args.get("current_user_id") db = get_db_read_replica() with db.scoped_session() as session: def get_unpopulated_playlists(): playlist_query = (session.query(Playlist).filter( Playlist.is_current == True)) # playlist ids filter if the optional query param is passed in if "playlist_id" in args: playlist_id_list = args.get("playlist_id") try: playlist_query = playlist_query.filter( Playlist.playlist_id.in_(playlist_id_list)) except ValueError as e: raise exceptions.ArgumentError( "Invalid value found in playlist id list", e) if "user_id" in args: user_id = args.get("user_id") # user id filter if the optional query param is passed in playlist_query = playlist_query.filter( Playlist.playlist_owner_id == user_id) # If no current_user_id, never show hidden playlists if not current_user_id: playlist_query = playlist_query.filter( Playlist.is_private == False) # Filter out deletes unless we're fetching explicitly by id if "playlist_id" not in args: playlist_query = playlist_query.filter( Playlist.is_delete == False) playlist_query = playlist_query.order_by(desc(Playlist.created_at)) playlists = paginate_query(playlist_query).all() playlists = helpers.query_result_to_list(playlists) # if we passed in a current_user_id, filter out all privte playlists where # the owner_id doesn't match the current_user_id if current_user_id: playlists = list( filter( lambda playlist: (not playlist["is_private"]) or playlist["playlist_owner_id"] == current_user_id, playlists)) # retrieve playlist ids list playlist_ids = list( map(lambda playlist: playlist["playlist_id"], playlists)) return (playlists, playlist_ids) try: # Get unpopulated playlists, either via # redis cache or via get_unpopulated_playlists key = make_cache_key(args) (playlists, playlist_ids) = use_redis_cache( key, UNPOPULATED_PLAYLIST_CACHE_DURATION_SEC, get_unpopulated_playlists) # bundle peripheral info into playlist results playlists = populate_playlist_metadata( session, playlist_ids, playlists, [RepostType.playlist, RepostType.album], [SaveType.playlist, SaveType.album], current_user_id) if args.get("with_users", False): user_id_list = get_users_ids(playlists) users = get_users_by_id(session, user_id_list, current_user_id) for playlist in playlists: user = users[playlist['playlist_owner_id']] if user: playlist['user'] = user except sqlalchemy.orm.exc.NoResultFound: pass return playlists
def get_tracks(args): """ Gets tracks. A note on caching strategy: - This method is cached at two layers: at the API via the @cache decorator, and within this method using the shared get_unpopulated_tracks cache. The shared cache only works when fetching via ID, so calls to fetch tracks via handle, asc/desc sort, or filtering by block_number won't hit the shared cache. These will hit the API cache unless they have a current_user_id included. """ tracks = [] db = get_db_read_replica() with db.scoped_session() as session: def get_tracks_and_ids(): if "handle" in args: handle = args.get("handle") user_id = session.query(User.user_id).filter( User.handle_lc == handle.lower()).first() args["user_id"] = user_id can_use_shared_cache = ("id" in args and not "min_block_number" in args and not "sort" in args and not "user_id" in args) if can_use_shared_cache: should_filter_deleted = args.get("filter_deleted", False) tracks = get_unpopulated_tracks(session, args["id"], should_filter_deleted) track_ids = list(map(lambda track: track["track_id"], tracks)) return (tracks, track_ids) (limit, offset) = get_pagination_vars() args["limit"] = limit args["offset"] = offset tracks = _get_tracks(session, args) track_ids = list(map(lambda track: track["track_id"], tracks)) return (tracks, track_ids) (tracks, track_ids) = get_tracks_and_ids() # bundle peripheral info into track results current_user_id = args.get("current_user_id") tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) if args.get("with_users", False): user_id_list = get_users_ids(tracks) users = get_users_by_id(session, user_id_list, current_user_id) for track in tracks: user = users[track['owner_id']] if user: track['user'] = user return tracks
def get_repost_feed_for_user(user_id, args): feed_results = {} db = get_db_read_replica() with db.scoped_session() as session: # query all reposts by user repost_query = (session.query(Repost).filter( Repost.is_current == True, Repost.is_delete == False, Repost.user_id == user_id).order_by(desc(Repost.created_at), desc(Repost.repost_item_id), desc(Repost.repost_type))) reposts = paginate_query(repost_query).all() # get track reposts from above track_reposts = [ r for r in reposts if r.repost_type == RepostType.track ] # get reposted track ids repost_track_ids = [r.repost_item_id for r in track_reposts] # get playlist reposts from above playlist_reposts = [ r for r in reposts if r.repost_type == RepostType.playlist or r.repost_type == RepostType.album ] # get reposted playlist ids repost_playlist_ids = [r.repost_item_id for r in playlist_reposts] track_reposts = helpers.query_result_to_list(track_reposts) playlist_reposts = helpers.query_result_to_list(playlist_reposts) # build track/playlist id --> repost dict from repost lists track_repost_dict = { repost["repost_item_id"]: repost for repost in track_reposts } playlist_repost_dict = { repost["repost_item_id"]: repost for repost in playlist_reposts } # query tracks for repost_track_ids track_query = (session.query(Track).filter( Track.is_current == True, Track.is_delete == False, Track.is_unlisted == False, Track.stem_of == None, Track.track_id.in_(repost_track_ids)).order_by( desc(Track.created_at))) tracks = track_query.all() tracks = helpers.query_result_to_list(tracks) # get track ids track_ids = [track["track_id"] for track in tracks] # query playlists for repost_playlist_ids playlist_query = (session.query(Playlist).filter( Playlist.is_current == True, Playlist.is_delete == False, Playlist.is_private == False, Playlist.playlist_id.in_(repost_playlist_ids)).order_by( desc(Playlist.created_at))) playlists = playlist_query.all() playlists = helpers.query_result_to_list(playlists) # get playlist ids playlist_ids = [playlist["playlist_id"] for playlist in playlists] # get repost counts by track and playlist IDs repost_counts = get_repost_counts(session, False, True, track_ids + playlist_ids, None) track_repost_counts = { repost_item_id: repost_count for (repost_item_id, repost_count, repost_type) in repost_counts if repost_type == RepostType.track } playlist_repost_counts = { repost_item_id: repost_count for (repost_item_id, repost_count, repost_type) in repost_counts if repost_type in (RepostType.playlist, RepostType.album) } # get save counts for tracks and playlists save_counts = get_save_counts(session, False, True, track_ids + playlist_ids, None) track_save_counts = { save_item_id: save_count for (save_item_id, save_count, save_type) in save_counts if save_type == SaveType.track } playlist_save_counts = { save_item_id: save_count for (save_item_id, save_count, save_type) in save_counts if save_type in (SaveType.playlist, SaveType.album) } current_user_id = get_current_user_id(required=False) requested_user_is_current_user = False user_reposted_track_ids = {} user_reposted_playlist_ids = {} user_saved_track_dict = {} user_saved_playlist_dict = {} followees_track_repost_dict = {} followees_playlist_repost_dict = {} if current_user_id: # if current user = user_id, skip current_user_reposted queries and default to true if current_user_id == user_id: requested_user_is_current_user = True else: user_reposted_query = (session.query( Repost.repost_item_id, Repost.repost_type).filter( Repost.is_current == True, Repost.is_delete == False, Repost.user_id == current_user_id, or_(Repost.repost_item_id.in_(track_ids), Repost.repost_item_id.in_(playlist_ids))).all()) # generate dictionary of track id --> current user reposted status user_reposted_track_ids = { r[0]: True for r in user_reposted_query if r[1] == RepostType.track } # generate dictionary of playlist id --> current user reposted status user_reposted_playlist_ids = { r[0]: True for r in user_reposted_query if r[1] == RepostType.album or r[1] == RepostType.playlist } # build dict of tracks and playlists that current user has saved # - query saves by current user from relevant tracks/playlists user_saved_query = (session.query( Save.save_item_id, Save.save_type).filter( Save.is_current == True, Save.is_delete == False, Save.user_id == current_user_id, or_(Save.save_item_id.in_(track_ids), Save.save_item_id.in_(playlist_ids))).all()) # - build dict of track id --> current user save status user_saved_track_dict = { save[0]: True for save in user_saved_query if save[1] == SaveType.track } # - build dict of playlist id --> current user save status user_saved_playlist_dict = { save[0]: True for save in user_saved_query if save[1] == SaveType.playlist or save[1] == SaveType.album } # query current user's followees followee_user_ids = (session.query(Follow.followee_user_id).filter( Follow.follower_user_id == current_user_id, Follow.is_current == True, Follow.is_delete == False).all()) followee_user_ids = [f[0] for f in followee_user_ids] # query all followees' reposts followee_repost_query = (session.query(Repost).filter( Repost.is_current == True, Repost.is_delete == False, Repost.user_id.in_(followee_user_ids), or_(Repost.repost_item_id.in_(repost_track_ids), Repost.repost_item_id.in_(repost_playlist_ids))).order_by( desc(Repost.created_at))) followee_reposts = paginate_query(followee_repost_query).all() followee_reposts = helpers.query_result_to_list(followee_reposts) # build dict of track id --> reposts from followee track reposts for repost in followee_reposts: if repost["repost_type"] == RepostType.track: if repost[ "repost_item_id"] not in followees_track_repost_dict: followees_track_repost_dict[ repost["repost_item_id"]] = [] followees_track_repost_dict[ repost["repost_item_id"]].append(repost) # build dict of playlist id --> reposts from followee playlist reposts for repost in followee_reposts: if (repost["repost_type"] == RepostType.playlist or repost["repost_type"] == RepostType.album): if repost[ "repost_item_id"] not in followees_playlist_repost_dict: followees_playlist_repost_dict[ repost["repost_item_id"]] = [] followees_playlist_repost_dict[ repost["repost_item_id"]].append(repost) # populate metadata for track entries for track in tracks: track[response_name_constants. repost_count] = track_repost_counts.get( track["track_id"], 0) track[response_name_constants.save_count] = track_save_counts.get( track["track_id"], 0) track[response_name_constants.has_current_user_reposted] = ( True if requested_user_is_current_user else user_reposted_track_ids.get(track["track_id"], False)) track[response_name_constants. has_current_user_saved] = user_saved_track_dict.get( track["track_id"], False) track[response_name_constants. followee_reposts] = followees_track_repost_dict.get( track["track_id"], []) track[response_name_constants. activity_timestamp] = track_repost_dict[ track["track_id"]]["created_at"] for playlist in playlists: playlist[response_name_constants. repost_count] = playlist_repost_counts.get( playlist["playlist_id"], 0) playlist[ response_name_constants.save_count] = playlist_save_counts.get( playlist["playlist_id"], 0) playlist[response_name_constants.has_current_user_reposted] = ( True if requested_user_is_current_user else user_reposted_playlist_ids.get(playlist["playlist_id"], False)) playlist[response_name_constants.has_current_user_saved] = \ user_saved_playlist_dict.get(playlist["playlist_id"], False) playlist[response_name_constants.followee_reposts] = \ followees_playlist_repost_dict.get(playlist["playlist_id"], []) playlist[response_name_constants.activity_timestamp] = \ playlist_repost_dict[playlist["playlist_id"]]["created_at"] unsorted_feed = tracks + playlists # sort feed by repost timestamp desc feed_results = sorted(unsorted_feed, key=lambda entry: entry[response_name_constants. activity_timestamp], reverse=True) if args.get("with_users", False): user_id_list = get_users_ids(feed_results) users = get_users_by_id(session, user_id_list) for result in feed_results: if 'playlist_owner_id' in result: user = users[result['playlist_owner_id']] if user: result['user'] = user elif 'owner_id' in result: user = users[result['owner_id']] if user: result['user'] = user return feed_results
def get_feed(args): feed_results = [] db = get_db_read_replica() feed_filter = args.get("filter") # Allow for fetching only tracks tracks_only = args.get('tracks_only', False) # Current user - user for whom feed is being generated current_user_id = get_current_user_id() with db.scoped_session() as session: # Generate list of users followed by current user, i.e. 'followees' followee_user_ids = (session.query(Follow.followee_user_id).filter( Follow.follower_user_id == current_user_id, Follow.is_current == True, Follow.is_delete == False).all()) followee_user_ids = [f[0] for f in followee_user_ids] # Fetch followee creations if requested if feed_filter in ["original", "all"]: if not tracks_only: # Query playlists posted by followees, sorted and paginated by created_at desc created_playlists_query = (session.query(Playlist).filter( Playlist.is_current == True, Playlist.is_delete == False, Playlist.is_private == False, Playlist.playlist_owner_id.in_( followee_user_ids)).order_by(desc( Playlist.created_at))) created_playlists = paginate_query(created_playlists_query, False).all() # get track ids for all tracks in playlists playlist_track_ids = set() for playlist in created_playlists: for track in playlist.playlist_contents["track_ids"]: playlist_track_ids.add(track["track"]) # get all track objects for track ids playlist_tracks = get_unpopulated_tracks( session, playlist_track_ids) playlist_tracks_dict = { track["track_id"]: track for track in playlist_tracks } # get all track ids that have same owner as playlist and created in "same action" # "same action": track created within [x time] before playlist creation tracks_to_dedupe = set() for playlist in created_playlists: for track_entry in playlist.playlist_contents["track_ids"]: track = playlist_tracks_dict.get(track_entry["track"]) if not track: return api_helpers.error_response( "Something caused the server to crash.") max_timedelta = datetime.timedelta( minutes=trackDedupeMaxMinutes) if (track["owner_id"] == playlist.playlist_owner_id) and \ (track["created_at"] <= playlist.created_at) and \ (playlist.created_at - track["created_at"] <= max_timedelta): tracks_to_dedupe.add(track["track_id"]) tracks_to_dedupe = list(tracks_to_dedupe) else: # No playlists to consider tracks_to_dedupe = [] created_playlists = [] # Query tracks posted by followees, sorted & paginated by created_at desc # exclude tracks that were posted in "same action" as playlist created_tracks_query = (session.query(Track).filter( Track.is_current == True, Track.is_delete == False, Track.is_unlisted == False, Track.stem_of == None, Track.owner_id.in_(followee_user_ids), Track.track_id.notin_(tracks_to_dedupe)).order_by( desc(Track.created_at))) created_tracks = paginate_query(created_tracks_query, False).all() # extract created_track_ids and created_playlist_ids created_track_ids = [track.track_id for track in created_tracks] created_playlist_ids = [ playlist.playlist_id for playlist in created_playlists ] # Fetch followee reposts if requested if feed_filter in ["repost", "all"]: # query items reposted by followees, sorted by oldest followee repost of item; # paginated by most recent repost timestamp repost_subquery = (session.query(Repost).filter( Repost.is_current == True, Repost.is_delete == False, Repost.user_id.in_(followee_user_ids))) # exclude items also created by followees to guarantee order determinism, in case of "all" filter if feed_filter == "all": repost_subquery = (repost_subquery.filter( or_( and_(Repost.repost_type == RepostType.track, Repost.repost_item_id.notin_(created_track_ids)), and_( Repost.repost_type != RepostType.track, Repost.repost_item_id.notin_( created_playlist_ids))))) repost_subquery = repost_subquery.subquery() repost_query = (session.query( repost_subquery.c.repost_item_id, repost_subquery.c.repost_type, func.min(repost_subquery.c.created_at).label( "min_created_at")).group_by( repost_subquery.c.repost_item_id, repost_subquery.c.repost_type).order_by( desc("min_created_at"))) followee_reposts = paginate_query(repost_query, False).all() # build dict of track_id / playlist_id -> oldest followee repost timestamp from followee_reposts above track_repost_timestamp_dict = {} playlist_repost_timestamp_dict = {} for (repost_item_id, repost_type, oldest_followee_repost_timestamp) in followee_reposts: if repost_type == RepostType.track: track_repost_timestamp_dict[ repost_item_id] = oldest_followee_repost_timestamp elif repost_type in (RepostType.playlist, RepostType.album): playlist_repost_timestamp_dict[ repost_item_id] = oldest_followee_repost_timestamp # extract reposted_track_ids and reposted_playlist_ids reposted_track_ids = list(track_repost_timestamp_dict.keys()) reposted_playlist_ids = list(playlist_repost_timestamp_dict.keys()) # Query tracks reposted by followees reposted_tracks = session.query(Track).filter( Track.is_current == True, Track.is_delete == False, Track.is_unlisted == False, Track.stem_of == None, Track.track_id.in_(reposted_track_ids)) # exclude tracks already fetched from above, in case of "all" filter if feed_filter == "all": reposted_tracks = reposted_tracks.filter( Track.track_id.notin_(created_track_ids)) reposted_tracks = reposted_tracks.order_by(desc( Track.created_at)).all() if not tracks_only: # Query playlists reposted by followees, excluding playlists already fetched from above reposted_playlists = session.query(Playlist).filter( Playlist.is_current == True, Playlist.is_delete == False, Playlist.is_private == False, Playlist.playlist_id.in_(reposted_playlist_ids)) # exclude playlists already fetched from above, in case of "all" filter if feed_filter == "all": reposted_playlists = reposted_playlists.filter( Playlist.playlist_id.notin_(created_playlist_ids)) reposted_playlists = reposted_playlists.order_by( desc(Playlist.created_at)).all() else: reposted_playlists = [] if feed_filter == "original": tracks_to_process = created_tracks playlists_to_process = created_playlists elif feed_filter == "repost": tracks_to_process = reposted_tracks playlists_to_process = reposted_playlists else: tracks_to_process = created_tracks + reposted_tracks playlists_to_process = created_playlists + reposted_playlists tracks = helpers.query_result_to_list(tracks_to_process) playlists = helpers.query_result_to_list(playlists_to_process) # define top level feed activity_timestamp to enable sorting # activity_timestamp: created_at if item created by followee, else reposted_at for track in tracks: if track["owner_id"] in followee_user_ids: track[response_name_constants. activity_timestamp] = track["created_at"] else: track[response_name_constants. activity_timestamp] = track_repost_timestamp_dict[ track["track_id"]] for playlist in playlists: if playlist["playlist_owner_id"] in followee_user_ids: playlist[response_name_constants. activity_timestamp] = playlist["created_at"] else: playlist[response_name_constants.activity_timestamp] = \ playlist_repost_timestamp_dict[playlist["playlist_id"]] # bundle peripheral info into track and playlist objects track_ids = list(map(lambda track: track["track_id"], tracks)) playlist_ids = list( map(lambda playlist: playlist["playlist_id"], playlists)) tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) playlists = populate_playlist_metadata( session, playlist_ids, playlists, [RepostType.playlist, RepostType.album], [SaveType.playlist, SaveType.album], current_user_id) # build combined feed of tracks and playlists unsorted_feed = tracks + playlists # sort feed based on activity_timestamp sorted_feed = sorted(unsorted_feed, key=lambda entry: entry[response_name_constants. activity_timestamp], reverse=True) # truncate feed to requested limit (limit, _) = get_pagination_vars() feed_results = sorted_feed[0:limit] if "with_users" in args and args.get("with_users") != 'false': user_id_list = get_users_ids(feed_results) users = get_users_by_id(session, user_id_list) for result in feed_results: if 'playlist_owner_id' in result: user = users[result['playlist_owner_id']] if user: result['user'] = user elif 'owner_id' in result: user = users[result['owner_id']] if user: result['user'] = user return feed_results
def _get_repost_feed_for_user(session: Session, user_id: int, args: GetRepostFeedForUserArgs): feed_results = [] current_user_id = args.get("current_user_id") limit = args.get("limit") offset = args.get("offset") if "handle" in args: handle = args.get("handle") or "" user_id = cast( int, session.query( User.user_id).filter(User.handle_lc == handle.lower()).first(), ) # Query all reposts by a user. # Outerjoin both tracks and playlists to collect both # so that a single limit/offset pagination does what we intend when tracks or playlists # are deleted. repost_query = ( session.query(Repost, Track, Playlist).outerjoin( Track, and_( Repost.repost_item_id == Track.track_id, Repost.repost_type == "track", Track.is_current == True, Track.is_delete == False, Track.is_unlisted == False, Track.stem_of == None, ), ).outerjoin( Playlist, and_( Repost.repost_item_id == Playlist.playlist_id, or_(Repost.repost_type == "playlist", Repost.repost_type == "album"), Playlist.is_current == True, Playlist.is_delete == False, Playlist.is_private == False, ), ).filter( Repost.is_current == True, Repost.is_delete == False, Repost.user_id == user_id, # Drop rows that have no join found for either track or playlist or_(Track.track_id != None, Playlist.playlist_id != None), ).order_by( desc(Repost.created_at), desc(Repost.repost_item_id), desc(Repost.repost_type), )) reposts = add_query_pagination(repost_query, limit, offset).all() # get track reposts from above track_reposts = [r[0] for r in reposts if r[1] is not None] track_reposts = helpers.query_result_to_list(track_reposts) # get playlist reposts from above playlist_reposts = [r[0] for r in reposts if r[2] is not None] playlist_reposts = helpers.query_result_to_list(playlist_reposts) # build track/playlist id --> repost dict from repost lists track_repost_dict = { repost["repost_item_id"]: repost for repost in track_reposts } playlist_repost_dict = { repost["repost_item_id"]: repost for repost in playlist_reposts } tracks = helpers.query_result_to_list( filter(None, [repost[1] for repost in reposts])) playlists = helpers.query_result_to_list( filter(None, [repost[2] for repost in reposts])) # get track ids track_ids = [track["track_id"] for track in tracks] # get playlist ids playlist_ids = [playlist["playlist_id"] for playlist in playlists] # populate full metadata tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) playlists = populate_playlist_metadata( session, playlist_ids, playlists, [RepostType.playlist, RepostType.album], [SaveType.playlist, SaveType.album], current_user_id, ) # add activity timestamps for track in tracks: track[response_name_constants.activity_timestamp] = track_repost_dict[ track["track_id"]]["created_at"] for playlist in playlists: playlist[ response_name_constants.activity_timestamp] = playlist_repost_dict[ playlist["playlist_id"]]["created_at"] unsorted_feed = tracks + playlists # sort feed by repost timestamp desc feed_results = sorted( unsorted_feed, key=lambda entry: entry[response_name_constants.activity_timestamp], reverse=True, ) if args.get("with_users", False): user_id_list = get_users_ids(feed_results) users = get_users_by_id(session, user_id_list) for result in feed_results: if "playlist_owner_id" in result: user = users[result["playlist_owner_id"]] if user: result["user"] = user elif "owner_id" in result: user = users[result["owner_id"]] if user: result["user"] = user return feed_results
def get_trending_playlists(args, strategy): """Returns Trending Playlists. Checks Redis cache for unpopulated playlists.""" db = get_db_read_replica() with db.scoped_session() as session: current_user_id = args.get("current_user_id", None) with_tracks = args.get("with_tracks", False) time = args.get("time") limit, offset = args.get("limit"), args.get("offset") key = make_trending_cache_key(time, strategy.version) # Get unpopulated playlists, # cached if it exists. (playlists, playlist_ids) = use_redis_cache( key, None, make_get_unpopulated_playlists(session, time, strategy)) # Apply limit + offset early to reduce the amount of # population work we have to do if limit is not None and offset is not None: playlists = playlists[offset:limit + offset] playlist_ids = playlist_ids[offset:limit + offset] # Populate playlist metadata playlists = populate_playlist_metadata( session, playlist_ids, playlists, [RepostType.playlist, RepostType.album], [SaveType.playlist, SaveType.album], current_user_id) trimmed_track_ids = None for playlist in playlists: playlist["track_count"] = len(playlist["tracks"]) playlist["tracks"] = playlist["tracks"][:PLAYLIST_TRACKS_LIMIT] # Trim track_ids, which ultimately become added_timestamps # and need to match the tracks. trimmed_track_ids = { track["track_id"] for track in playlist["tracks"] } playlist_track_ids = playlist["playlist_contents"]["track_ids"] playlist_track_ids = list( filter(lambda track_id: track_id["track"] in trimmed_track_ids, playlist_track_ids)) playlist["playlist_contents"]["track_ids"] = playlist_track_ids playlists_map = { playlist['playlist_id']: playlist for playlist in playlists } if with_tracks: # populate track metadata tracks = [] for playlist in playlists: playlist_tracks = playlist["tracks"] tracks.extend(playlist_tracks) track_ids = [track["track_id"] for track in tracks] populated_tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) # Add users if necessary add_users_to_tracks(session, populated_tracks, current_user_id) # Re-associate tracks with playlists # track_id -> populated_track populated_track_map = { track["track_id"]: track for track in populated_tracks } for playlist in playlists_map.values(): for i in range(len(playlist["tracks"])): track_id = playlist["tracks"][i]["track_id"] populated = populated_track_map[track_id] playlist["tracks"][i] = populated playlist["tracks"] = list(map(extend_track, playlist["tracks"])) # re-sort playlists to original order, because populate_playlist_metadata # unsorts. sorted_playlists = [ playlists_map[playlist_id] for playlist_id in playlist_ids ] # Add users to playlists user_id_list = get_users_ids(sorted_playlists) users = get_users_by_id(session, user_id_list, current_user_id) for playlist in sorted_playlists: user = users[playlist['playlist_owner_id']] if user: playlist['user'] = user # Extend the playlists playlists = list(map(extend_playlist, playlists)) return sorted_playlists
def get_top_playlists(kind, args): current_user_id = get_current_user_id(required=False) # Argument parsing and checking if kind not in ("playlist", "album"): raise exceptions.ArgumentError( "Invalid kind provided, must be one of 'playlist', 'album'" ) limit = args.get("limit", 16) mood = args.get("mood", None) if "filter" in args: query_filter = args.get("filter") if query_filter != "followees": raise exceptions.ArgumentError( "Invalid filter provided, must be one of 'followees'" ) if query_filter == "followees": if not current_user_id: raise exceptions.ArgumentError( "User id required to query for followees" ) else: query_filter = None db = get_db_read_replica() with db.scoped_session() as session: # If filtering by followees, set the playlist view to be only playlists from # users that the current user follows. if query_filter == "followees": playlists_to_query = create_followee_playlists_subquery( session, current_user_id ) # Otherwise, just query all playlists else: playlists_to_query = session.query(Playlist).subquery() # Create a decayed-score view of the playlists playlist_query = ( session.query( playlists_to_query, (AggregatePlaylist.repost_count + AggregatePlaylist.save_count).label( "count" ), decayed_score( AggregatePlaylist.repost_count + AggregatePlaylist.save_count, playlists_to_query.c.created_at, ).label("score"), ) .select_from(playlists_to_query) .join( AggregatePlaylist, AggregatePlaylist.playlist_id == playlists_to_query.c.playlist_id, ) .filter( playlists_to_query.c.is_current == True, playlists_to_query.c.is_delete == False, playlists_to_query.c.is_private == False, playlists_to_query.c.is_album == (kind == "album"), ) ) # Filter by mood (no-op if no mood is provided) playlist_query = filter_to_playlist_mood( session, mood, playlist_query, playlists_to_query ) # Order and limit the playlist query by score playlist_query = playlist_query.order_by( desc("score"), desc(playlists_to_query.c.playlist_id) ).limit(limit) playlist_results = playlist_query.all() # Unzip query results into playlists and scores score_map = {} # playlist_id : score playlists = [] if playlist_results: for result in playlist_results: # The playlist is the portion of the query result before repost_count and score playlist = result[0:-2] score = result[-1] # Convert the playlist row tuple into a dictionary keyed by column name playlist = helpers.tuple_to_model_dictionary(playlist, Playlist) score_map[playlist["playlist_id"]] = score playlists.append(playlist) playlist_ids = list(map(lambda playlist: playlist["playlist_id"], playlists)) # Bundle peripheral info into playlist results playlists = populate_playlist_metadata( session, playlist_ids, playlists, [RepostType.playlist, RepostType.album], [SaveType.playlist, SaveType.album], current_user_id, ) # Add scores into the response for playlist in playlists: playlist["score"] = score_map[playlist["playlist_id"]] if args.get("with_users", False): user_id_list = get_users_ids(playlists) users = get_users_by_id(session, user_id_list) for playlist in playlists: user = users[playlist["playlist_owner_id"]] if user: playlist["user"] = user return playlists
def get_tracks(args): tracks = [] db = get_db_read_replica() with db.scoped_session() as session: # Create initial query base_query = session.query(Track) base_query = base_query.filter(Track.is_current == True, Track.is_unlisted == False, Track.stem_of == None) # Conditionally process an array of tracks if "id" in args: track_id_list = args.get("id") try: # Update query with track_id list base_query = base_query.filter( Track.track_id.in_(track_id_list)) except ValueError as e: logger.error("Invalid value found in track id list", exc_info=True) raise e # Allow filtering of tracks by a certain creator if "user_id" in args: user_id = args.get("user_id") base_query = base_query.filter(Track.owner_id == user_id) # Allow filtering of deletes if "filter_deleted" in args: filter_deleted = args.get("filter_deleted") if filter_deleted: base_query = base_query.filter(Track.is_delete == False) if "min_block_number" in args: min_block_number = args.get("min_block_number") base_query = base_query.filter( Track.blocknumber >= min_block_number) whitelist_params = [ 'created_at', 'create_date', 'release_date', 'blocknumber', 'track_id' ] base_query = parse_sort_param(base_query, Track, whitelist_params) query_results = paginate_query(base_query).all() tracks = helpers.query_result_to_list(query_results) track_ids = list(map(lambda track: track["track_id"], tracks)) current_user_id = get_current_user_id(required=False) # bundle peripheral info into track results tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) if args.get("with_users", False): user_id_list = get_users_ids(tracks) users = get_users_by_id(session, user_id_list) for track in tracks: user = users[track['owner_id']] if user: track['user'] = user return tracks
def get_tracks_including_unlisted(args): """Fetch a track, allowing unlisted. Args: args: dict args.identifiers: array of { handle, id, url_title} dicts args.current_user_id: optional current user ID args.filter_deleted: filter deleted tracks args.with_users: include users in unlisted tracks """ tracks = [] identifiers = args["identifiers"] for i in identifiers: helpers.validate_arguments(i, ["handle", "id", "url_title"]) current_user_id = args.get("current_user_id") db = get_db_read_replica() with db.scoped_session() as session: def get_unpopulated_track(): base_query = session.query(Track) filter_cond = [] # Create filter conditions as a list of `and` clauses for i in identifiers: filter_cond.append( and_(Track.is_current == True, Track.track_id == i["id"])) # Pass array of `and` clauses into an `or` clause as destructured *args base_query = base_query.filter(or_(*filter_cond)) # Allow filtering of deletes # Note: There is no standard for boolean url parameters, and any value (including 'false') # will be evaluated as true, so an explicit check is made for true if "filter_deleted" in args: filter_deleted = args.get("filter_deleted") if filter_deleted: base_query = base_query.filter(Track.is_delete == False) # Perform the query # TODO: pagination is broken with unlisted tracks query_results = paginate_query(base_query).all() tracks = helpers.query_result_to_list(query_results) # Mapping of track_id -> track object from request; # used to check route_id when iterating through identifiers identifiers_map = {track["id"]: track for track in identifiers} # If the track is unlisted and the generated route_id does not match the route_id in db, # filter track out from response def filter_fn(track): input_track = identifiers_map[track["track_id"]] route_id = helpers.create_track_route_id( input_track["url_title"], input_track["handle"]) return not track["is_unlisted"] or track["route_id"] == route_id tracks = list(filter(filter_fn, tracks)) track_ids = list(map(lambda track: track["track_id"], tracks)) return (tracks, track_ids) key = make_cache_key(args) (tracks, track_ids) = use_redis_cache(key, UNPOPULATED_TRACK_CACHE_DURATION_SEC, get_unpopulated_track) # Add users if args.get("with_users", False): user_id_list = get_users_ids(tracks) users = get_users_by_id(session, user_id_list, current_user_id) for track in tracks: user = users[track["owner_id"]] if user: track["user"] = user # Populate metadata tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) return tracks
def get_top_followee_windowed(type, window, args): if type != 'track': raise exceptions.ArgumentError( "Invalid type provided, must be one of 'track'" ) valid_windows = ['week', 'month', 'year'] if not window or window not in valid_windows: raise exceptions.ArgumentError( "Invalid window provided, must be one of {}".format(valid_windows) ) limit = args.get('limit', 25) current_user_id = get_current_user_id() db = get_db_read_replica() with db.scoped_session() as session: # Construct a subquery to get the summed save + repost count for the `type` count_subquery = create_save_repost_count_subquery(session, type) followee_user_ids = ( session.query(Follow.followee_user_id) .filter( Follow.follower_user_id == current_user_id, Follow.is_current == True, Follow.is_delete == False ) ) followee_user_ids_subquery = followee_user_ids.subquery() # Queries for tracks joined against followed users and counts tracks_query = ( session.query( Track, ) .join( followee_user_ids_subquery, Track.owner_id == followee_user_ids_subquery.c.followee_user_id ) .join( count_subquery, Track.track_id == count_subquery.c['id'] ) .filter( Track.is_current == True, Track.is_delete == False, Track.is_unlisted == False, Track.stem_of == None, # Query only tracks created `window` time ago (week, month, etc.) Track.created_at >= text( "NOW() - interval '1 {}'".format(window)), ) .order_by( desc(count_subquery.c['count']), desc(Track.track_id) ) .limit(limit) ) tracks_query_results = tracks_query.all() tracks = helpers.query_result_to_list(tracks_query_results) track_ids = list(map(lambda track: track['track_id'], tracks)) # Bundle peripheral info into track results tracks = populate_track_metadata( session, track_ids, tracks, current_user_id) if args.get('with_users', False): user_id_list = get_users_ids(tracks) users = get_users_by_id(session, user_id_list) for track in tracks: user = users[track['owner_id']] if user: track['user'] = user return tracks
def search(args): """Perform a search. `args` should contain `is_auto_complete`, `query`, `kind`, `current_user_id`, and `only_downloadable` """ if os.getenv("audius_elasticsearch_search_enabled"): try: resp = search_es_full(args) return resp except Exception as e: logger.error(f"Elasticsearch error: {e}") search_str = args.get("query") # when creating query table, we substitute this too search_str = search_str.replace("&", "and") kind = args.get("kind", "all") is_auto_complete = args.get("is_auto_complete") current_user_id = args.get("current_user_id") only_downloadable = args.get("only_downloadable") limit = args.get("limit") offset = args.get("offset") searchKind = SearchKind[kind] results = {} # Accumulate user_ids for later user_ids = set() # Create args for perform_search_query search_args = { "search_str": search_str, "limit": limit, "offset": offset, "is_auto_complete": is_auto_complete, "current_user_id": current_user_id, "only_downloadable": only_downloadable, } if search_str: db = get_db_read_replica() # Concurrency approach: # Spin up a ThreadPoolExecutor for each request to perform_search_query # to perform the different search types in parallel. # After each future resolves, we then add users for each entity in a single # db round trip. with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: # Keep a mapping of future -> search_type futures_map = {} futures = [] # Helper fn to submit a future and add it to bookkeeping data structures def submit_and_add(search_type): future = executor.submit( perform_search_query, db, search_type, search_args ) futures.append(future) futures_map[future] = search_type if searchKind in [SearchKind.all, SearchKind.tracks]: submit_and_add("tracks") if searchKind in [SearchKind.all, SearchKind.users]: submit_and_add("users") if searchKind in [SearchKind.all, SearchKind.playlists]: submit_and_add("playlists") if searchKind in [SearchKind.all, SearchKind.albums]: submit_and_add("albums") for future in concurrent.futures.as_completed(futures): search_result = future.result() future_type = futures_map[future] # Add to the final results # Add to user_ids if future_type == "tracks": results["tracks"] = search_result["all"] results["saved_tracks"] = search_result["saved"] elif future_type == "users": results["users"] = search_result["all"] results["followed_users"] = search_result["followed"] elif future_type == "playlists": results["playlists"] = search_result["all"] results["saved_playlists"] = search_result["saved"] elif future_type == "albums": results["albums"] = search_result["all"] results["saved_albums"] = search_result["saved"] user_ids.update(get_users_ids(search_result["all"])) with db.scoped_session() as session: # Add users back users = get_users_by_id(session, list(user_ids), current_user_id) for (_, result_list) in results.items(): for result in result_list: user_id = None if "playlist_owner_id" in result: user_id = result["playlist_owner_id"] elif "owner_id" in result: user_id = result["owner_id"] if user_id is not None: user = users[user_id] result["user"] = user return extend_search(results)
def get_tracks(args): """ Gets tracks. A note on caching strategy: - This method is cached at two layers: at the API via the @cache decorator, and within this method using the shared get_unpopulated_tracks cache. The shared cache only works when fetching via ID, so calls to fetch tracks via handle, asc/desc sort, or filtering by block_number won't hit the shared cache. These will hit the API cache unless they have a current_user_id included. """ tracks = [] db = get_db_read_replica() with db.scoped_session() as session: def get_tracks_and_ids(): if "handle" in args: handle = args.get("handle") user_id = session.query(User.user_id).filter( User.handle_lc == handle.lower()).first() args["user_id"] = user_id can_use_shared_cache = ("id" in args and not "min_block_number" in args and not "sort" in args and not "user_id" in args) if can_use_shared_cache: should_filter_deleted = args.get("filter_deleted", False) tracks = get_unpopulated_tracks(session, args["id"], should_filter_deleted) track_ids = list(map(lambda track: track["track_id"], tracks)) return (tracks, track_ids) # Create initial query base_query = session.query(Track) base_query = base_query.filter(Track.is_current == True, Track.is_unlisted == False, Track.stem_of == None) # Conditionally process an array of tracks if "id" in args: track_id_list = args.get("id") try: # Update query with track_id list base_query = base_query.filter( Track.track_id.in_(track_id_list)) except ValueError as e: logger.error("Invalid value found in track id list", exc_info=True) raise e # Allow filtering of tracks by a certain creator if "user_id" in args: user_id = args.get("user_id") base_query = base_query.filter(Track.owner_id == user_id) # Allow filtering of deletes if "filter_deleted" in args: filter_deleted = args.get("filter_deleted") if filter_deleted: base_query = base_query.filter(Track.is_delete == False) if "min_block_number" in args: min_block_number = args.get("min_block_number") base_query = base_query.filter( Track.blocknumber >= min_block_number) if "sort" in args: if args["sort"] == "date": base_query = base_query.order_by(Track.created_at.desc(), Track.track_id.desc()) elif args["sort"] == "plays": base_query = base_query.join( AggregatePlays, AggregatePlays.play_item_id == Track.track_id).order_by(AggregatePlays.count.desc()) else: whitelist_params = [ 'created_at', 'create_date', 'release_date', 'blocknumber', 'track_id' ] base_query = parse_sort_param(base_query, Track, whitelist_params) query_results = paginate_query(base_query).all() tracks = helpers.query_result_to_list(query_results) track_ids = list(map(lambda track: track["track_id"], tracks)) return (tracks, track_ids) (tracks, track_ids) = get_tracks_and_ids() # bundle peripheral info into track results current_user_id = args.get("current_user_id") tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) if args.get("with_users", False): user_id_list = get_users_ids(tracks) users = get_users_by_id(session, user_id_list, current_user_id) for track in tracks: user = users[track['owner_id']] if user: track['user'] = user return tracks
def get_repost_feed_for_user(user_id, args): feed_results = {} db = get_db_read_replica() current_user_id = args.get("current_user_id") with db.scoped_session() as session: if "handle" in args: handle = args.get("handle") user_id = session.query( User.user_id).filter(User.handle_lc == handle.lower()).first() # query all reposts by user repost_query = (session.query(Repost).filter( Repost.is_current == True, Repost.is_delete == False, Repost.user_id == user_id).order_by(desc(Repost.created_at), desc(Repost.repost_item_id), desc(Repost.repost_type))) reposts = paginate_query(repost_query).all() # get track reposts from above track_reposts = [ r for r in reposts if r.repost_type == RepostType.track ] # get reposted track ids repost_track_ids = [r.repost_item_id for r in track_reposts] # get playlist reposts from above playlist_reposts = [ r for r in reposts if r.repost_type == RepostType.playlist or r.repost_type == RepostType.album ] # get reposted playlist ids repost_playlist_ids = [r.repost_item_id for r in playlist_reposts] track_reposts = helpers.query_result_to_list(track_reposts) playlist_reposts = helpers.query_result_to_list(playlist_reposts) # build track/playlist id --> repost dict from repost lists track_repost_dict = { repost["repost_item_id"]: repost for repost in track_reposts } playlist_repost_dict = { repost["repost_item_id"]: repost for repost in playlist_reposts } # query tracks for repost_track_ids track_query = (session.query(Track).filter( Track.is_current == True, Track.is_delete == False, Track.is_unlisted == False, Track.stem_of == None, Track.track_id.in_(repost_track_ids)).order_by( desc(Track.created_at))) tracks = track_query.all() tracks = helpers.query_result_to_list(tracks) # get track ids track_ids = [track["track_id"] for track in tracks] # query playlists for repost_playlist_ids playlist_query = (session.query(Playlist).filter( Playlist.is_current == True, Playlist.is_delete == False, Playlist.is_private == False, Playlist.playlist_id.in_(repost_playlist_ids)).order_by( desc(Playlist.created_at))) playlists = playlist_query.all() playlists = helpers.query_result_to_list(playlists) # get playlist ids playlist_ids = [playlist["playlist_id"] for playlist in playlists] # populate full metadata tracks = populate_track_metadata(session, track_ids, tracks, current_user_id) playlists = populate_playlist_metadata( session, playlist_ids, playlists, [RepostType.playlist, RepostType.album], [SaveType.playlist, SaveType.album], current_user_id) # add activity timestamps for track in tracks: track[response_name_constants. activity_timestamp] = track_repost_dict[ track["track_id"]]["created_at"] for playlist in playlists: playlist[response_name_constants.activity_timestamp] = \ playlist_repost_dict[playlist["playlist_id"]]["created_at"] unsorted_feed = tracks + playlists # sort feed by repost timestamp desc feed_results = sorted(unsorted_feed, key=lambda entry: entry[response_name_constants. activity_timestamp], reverse=True) if args.get("with_users", False): user_id_list = get_users_ids(feed_results) users = get_users_by_id(session, user_id_list) for result in feed_results: if 'playlist_owner_id' in result: user = users[result['playlist_owner_id']] if user: result['user'] = user elif 'owner_id' in result: user = users[result['owner_id']] if user: result['user'] = user return feed_results
def get_playlists(args): playlists = [] current_user_id = get_current_user_id(required=False) filter_out_private_playlists = True db = get_db_read_replica() with db.scoped_session() as session: try: playlist_query = ( session.query(Playlist) .filter(Playlist.is_current == True) ) # playlist ids filter if the optional query param is passed in if "playlist_id" in args: playlist_id_list = args.get("playlist_id") try: playlist_query = playlist_query.filter(Playlist.playlist_id.in_(playlist_id_list)) except ValueError as e: raise exceptions.ArgumentError("Invalid value found in playlist id list", e) if "user_id" in args: user_id = args.get("user_id") # user id filter if the optional query param is passed in playlist_query = playlist_query.filter( Playlist.playlist_owner_id == user_id ) # if the current user is the same as the user passed in through the query param then we're trying # to get playlists for, check if the users are the same. if they are the same, the current user is # trying to request their own playlists, so allow them to see private playlists if current_user_id and user_id and (int(current_user_id) == int(user_id)): filter_out_private_playlists = False if filter_out_private_playlists: playlist_query = playlist_query.filter( Playlist.is_private == False ) # Filter out deletes unless we're fetching explicitly by id if "playlist_id" not in args: playlist_query = playlist_query.filter( Playlist.is_delete == False ) playlist_query = playlist_query.order_by(desc(Playlist.created_at)) playlists = paginate_query(playlist_query).all() playlists = helpers.query_result_to_list(playlists) # retrieve playlist ids list playlist_ids = list(map(lambda playlist: playlist["playlist_id"], playlists)) current_user_id = get_current_user_id(required=False) # bundle peripheral info into playlist results playlists = populate_playlist_metadata( session, playlist_ids, playlists, [RepostType.playlist, RepostType.album], [SaveType.playlist, SaveType.album], current_user_id ) if args.get("with_users", False): user_id_list = get_users_ids(playlists) users = get_users_by_id(session, user_id_list) for playlist in playlists: user = users[playlist['playlist_owner_id']] if user: playlist['user'] = user except sqlalchemy.orm.exc.NoResultFound: pass return playlists