def first_unfiltered_item( self, items: List[Dict], get_title_and_artist: Callable[[Dict], Tuple[str, str]]) -> Optional[Dict]: """Returns the first item in the given list that is not filtered.""" for item in items: artist, title = get_title_and_artist(item) if song_utils.is_forbidden(artist) or song_utils.is_forbidden( title): continue return item self.error = "All results filtered" return None
def check_available(self) -> bool: # directly use the search extractors entry function so we can process each result # as soon as it's available instead of waiting for all of them extractor = youtube_dl.extractor.youtube.YoutubeSearchIE() extractor._downloader = youtube_dl.YoutubeDL(self.ydl_opts) extractor.initialize() for entry in extractor._entries(self.query, 50): if song_utils.is_forbidden(self.musiq, entry["title"]): continue try: with youtube_dl.YoutubeDL(self.ydl_opts) as ydl: self.info_dict = ydl.extract_info(entry["id"], download=False) break except ( youtube_dl.utils.ExtractorError, youtube_dl.utils.DownloadError, ) as e: logging.warning("error during availability check for %s:", entry["id"]) logging.warning(e) else: self.error = "All results filtered" return False self.id = self.info_dict["id"] return self.check_not_too_large(self.info_dict["filesize"])
def get_search_suggestions( self, musiq: Musiq, query: str, playlist: bool ) -> List[Tuple[str, str]]: """Returns a list of suggested items for the given query. Returns playlists if :param playlist: is True, songs otherwise.""" result = self.web_client.get( "search", params={ "q": query, "limit": "20", "market": "from_token", "type": "playlist" if playlist else "track", }, ) if playlist: items = result["playlists"]["items"] else: items = result["tracks"]["items"] suggestions = [] for item in items: external_url = item["external_urls"]["spotify"] title = item["name"] if playlist: displayname = title else: artist = item["artists"][0]["name"] # apply filter from the settings if song_utils.is_forbidden(musiq, artist) or song_utils.is_forbidden( musiq, title ): continue displayname = song_utils.displayname(artist, title) suggestions.append((displayname, external_url)) # remove duplicates and filter by keywords chosen_displaynames = set() unique_suggestions = [] for suggestion in suggestions: if suggestion[0] in chosen_displaynames: continue unique_suggestions.append(suggestion) chosen_displaynames.add(suggestion[0]) return unique_suggestions
def get_search_suggestions(self, query: str) -> List[str]: """Returns a list of suggested items for the given query.""" response = self.web_client.get( "https://api-v2.soundcloud.com/search/queries", q=query) suggestions = [ item.query for item in response.collection if not song_utils.is_forbidden(item.query) ] return suggestions
def gather_metadata(self) -> bool: """Fetches metadata for this song's uri from Spotify.""" if not self.id: results = self.web_client.get( "search", params={ "q": self.query, "limit": "50", "market": "from_token", "type": "track", }, ) # apply the filterlist from the settings for item in results["tracks"]["items"]: artist = item["artists"][0]["name"] title = item["name"] if song_utils.is_forbidden( self.musiq, artist ) or song_utils.is_forbidden(self.musiq, title): continue result = item break else: # all tracks got filtered self.error = "All results filtered" return False else: result = self.web_client.get(f"tracks/{self.id}", params={"limit": "1"}) try: self.metadata["internal_url"] = result["uri"] self.metadata["external_url"] = result["external_urls"]["spotify"] self.metadata["artist"] = result["artists"][0]["name"] self.metadata["title"] = result["name"] self.metadata["duration"] = result["duration_ms"] / 1000 except KeyError: self.error = "No song found" return False return True
def get_search_suggestions(musiq: Musiq, query: str) -> List[str]: """Returns a list of suggestions for the given query from Youtube.""" with youtube_session() as session: params = { "client": "youtube", "q": query[:100], # queries longer than 100 characters are not accepted "xhr": "t", # this makes the response be a json file } response = session.get( "https://clients1.google.com/complete/search", params=params ) suggestions = json.loads(response.text) # first entry is the query, the second one contains the suggestions suggestions = suggestions[1] # suggestions are given as tuples # extract the string and skip the query if it occurs identically suggestions = [ entry[0] for entry in suggestions if entry[0] != query and not song_utils.is_forbidden(musiq, entry[0]) ] return suggestions
def get_suggestions(self, request: WSGIRequest) -> JsonResponse: """Returns suggestions for a given query. Combines online and offline suggestions.""" query = request.GET["term"] suggest_playlist = request.GET["playlist"] == "true" if self.musiq.base.settings.basic.new_music_only and not suggest_playlist: return JsonResponse([], safe=False) results = self._online_suggestions(query, suggest_playlist) basic_settings = self.musiq.base.settings.basic if suggest_playlist: search_results = watson.search(query, models=( ArchivedPlaylist, ))[:basic_settings.number_of_suggestions] for playlist in search_results: playlist_info = playlist.meta archived_playlist = ArchivedPlaylist.objects.get( id=playlist_info["id"]) result_dict: Dict[str, Union[str, int]] = { "key": playlist_info["id"], "value": playlist_info["title"], "counter": playlist.object.counter, "type": song_utils.determine_playlist_type(archived_playlist), } results.append(result_dict) else: search_results = watson.search( query, models=(ArchivedSong, ))[:basic_settings.number_of_suggestions] for search_result in search_results: song_info = search_result.meta if song_utils.is_forbidden( self.musiq, song_info["artist"]) or song_utils.is_forbidden( self.musiq, song_info["title"]): continue try: provider = SongProvider.create( self.musiq, external_url=song_info["url"]) except NotImplementedError: # For this song a provider is necessary that is not available # e.g. the song was played before, but the provider was disabled continue cached = provider.check_cached() # don't suggest local songs if they are not cached (=not at expected location) if not cached and provider.type == "local": continue # don't suggest online songs when we don't have internet if not self.musiq.base.settings.basic.has_internet and not cached: continue # don't suggest youtube songs if it was disabled if (not self.musiq.base.settings.platforms.youtube_enabled and provider.type == "youtube"): continue # don't suggest spotify songs if we are not logged in if (not self.musiq.base.settings.platforms.spotify_enabled and provider.type == "spotify"): continue # don't suggest soundcloud songs if we are not logged in if (not self.musiq.base.settings.platforms.soundcloud_enabled and provider.type == "soundcloud"): continue result_dict = { "key": song_info["id"], "value": song_utils.displayname(song_info["artist"], song_info["title"]), "counter": search_result.object.counter, "type": provider.type, } results.append(result_dict) return JsonResponse(results, safe=False)
def _offline_song_suggestions(query: str) -> List[SuggestionResult]: results: List[SuggestionResult] = [] terms = query.split() song_results: Iterable[Mapping[str, Any]] if settings.DEBUG: # Testing the whole table whether it contains any term is quite costly. # Used for sqlite3 which does not have a similarity function. matching_songs = ArchivedSong.objects.prefetch_related("queries") for term in terms: matching_songs = matching_songs.filter( Q(title__icontains=term) | Q(artist__icontains=term) | Q(queries__query__icontains=term)) song_results = ( matching_songs.order_by("-counter") # annotate with same values as in the postgres case to have a consistent interface .annotate( u_id=F("id"), u_url=F("url"), u_artist=F("artist"), u_title=F("title"), u_duration=F("duration"), u_counter=F("counter"), u_cached=F("cached"), ).values(*u_values_list).distinct() [:storage.get("number_of_suggestions")]) # Perhaps this could be combined with the similarity search # to improve usability with the right weighting. # matching_songs = matching_songs.annotate( # artist_similarity=Coalesce(TrigramWordSimilarity(query, "artist"), 0), # title_similarity=Coalesce(TrigramWordSimilarity(query, "title"), 0), # query_similarity=Coalesce(TrigramWordSimilarity(query, "queries__query"), 0), # max_similarity=Greatest( # "artist_similarity", "title_similarity", "query_similarity" # ), # ) # To combine, use union instead of | (or) in order to access the annotated values # similar_songs = union(matching_songs) else: song_results = _postgres_song_results(query) has_internet = redis.get("has_internet") for song in song_results: if song_utils.is_forbidden( song["u_artist"]) or song_utils.is_forbidden(song["u_title"]): continue platform = song_utils.determine_url_type(song["u_url"]) # don't suggest online songs when we don't have internet if not has_internet and not song["u_cached"]: continue if platform == "local": # don't suggest local songs if they are not cached (=not at expected location) if not song["u_cached"]: continue else: # don't suggest songs if the respective platform is disabled assert platform in ["youtube", "spotify", "soundcloud", "jamendo"] if not storage.get(cast(PlatformEnabled, f"{platform}_enabled")): continue result_dict: SuggestionResult = { "key": song["u_id"], "value": song_utils.displayname(song["u_artist"], song["u_title"]), "counter": song["u_counter"], "type": platform, "durationFormatted": song_utils.format_seconds(song["u_duration"]), } results.append(result_dict) # mark suggestions whose displayname is identical seen_values: Dict[str, int] = {} for index, result in enumerate(results): if result["value"] in seen_values: result["confusable"] = True results[seen_values[result["value"]]]["confusable"] = True seen_values[result["value"]] = index return results