def first_unfiltered_item(
self, items: List[Dict],
get_title_and_artist: Callable[[Dict], Tuple[str,
str]]) -> Optional[Dict]:
"""Returns the first item in the given list that is not filtered."""
for item in items:
artist, title = get_title_and_artist(item)
if song_utils.is_forbidden(artist) or song_utils.is_forbidden(
title):
continue
return item
self.error = "All results filtered"
return None
def check_available(self) -> bool:
# directly use the search extractors entry function so we can process each result
# as soon as it's available instead of waiting for all of them
extractor = youtube_dl.extractor.youtube.YoutubeSearchIE()
extractor._downloader = youtube_dl.YoutubeDL(self.ydl_opts)
extractor.initialize()
for entry in extractor._entries(self.query, 50):
if song_utils.is_forbidden(self.musiq, entry["title"]):
continue
try:
with youtube_dl.YoutubeDL(self.ydl_opts) as ydl:
self.info_dict = ydl.extract_info(entry["id"], download=False)
break
except (
youtube_dl.utils.ExtractorError,
youtube_dl.utils.DownloadError,
) as e:
logging.warning("error during availability check for %s:", entry["id"])
logging.warning(e)
else:
self.error = "All results filtered"
return False
self.id = self.info_dict["id"]
return self.check_not_too_large(self.info_dict["filesize"])
def get_search_suggestions(
self, musiq: Musiq, query: str, playlist: bool
) -> List[Tuple[str, str]]:
"""Returns a list of suggested items for the given query.
Returns playlists if :param playlist: is True, songs otherwise."""
result = self.web_client.get(
"search",
params={
"q": query,
"limit": "20",
"market": "from_token",
"type": "playlist" if playlist else "track",
},
)
if playlist:
items = result["playlists"]["items"]
else:
items = result["tracks"]["items"]
suggestions = []
for item in items:
external_url = item["external_urls"]["spotify"]
title = item["name"]
if playlist:
displayname = title
else:
artist = item["artists"][0]["name"]
# apply filter from the settings
if song_utils.is_forbidden(musiq, artist) or song_utils.is_forbidden(
musiq, title
):
continue
displayname = song_utils.displayname(artist, title)
suggestions.append((displayname, external_url))
# remove duplicates and filter by keywords
chosen_displaynames = set()
unique_suggestions = []
for suggestion in suggestions:
if suggestion[0] in chosen_displaynames:
continue
unique_suggestions.append(suggestion)
chosen_displaynames.add(suggestion[0])
return unique_suggestions
def get_search_suggestions(self, query: str) -> List[str]:
"""Returns a list of suggested items for the given query."""
response = self.web_client.get(
"https://api-v2.soundcloud.com/search/queries", q=query)
suggestions = [
item.query for item in response.collection
if not song_utils.is_forbidden(item.query)
]
return suggestions
def gather_metadata(self) -> bool:
"""Fetches metadata for this song's uri from Spotify."""
if not self.id:
results = self.web_client.get(
"search",
params={
"q": self.query,
"limit": "50",
"market": "from_token",
"type": "track",
},
)
# apply the filterlist from the settings
for item in results["tracks"]["items"]:
artist = item["artists"][0]["name"]
title = item["name"]
if song_utils.is_forbidden(
self.musiq, artist
) or song_utils.is_forbidden(self.musiq, title):
continue
result = item
break
else:
# all tracks got filtered
self.error = "All results filtered"
return False
else:
result = self.web_client.get(f"tracks/{self.id}", params={"limit": "1"})
try:
self.metadata["internal_url"] = result["uri"]
self.metadata["external_url"] = result["external_urls"]["spotify"]
self.metadata["artist"] = result["artists"][0]["name"]
self.metadata["title"] = result["name"]
self.metadata["duration"] = result["duration_ms"] / 1000
except KeyError:
self.error = "No song found"
return False
return True
def get_search_suggestions(musiq: Musiq, query: str) -> List[str]:
"""Returns a list of suggestions for the given query from Youtube."""
with youtube_session() as session:
params = {
"client": "youtube",
"q": query[:100], # queries longer than 100 characters are not accepted
"xhr": "t", # this makes the response be a json file
}
response = session.get(
"https://clients1.google.com/complete/search", params=params
)
suggestions = json.loads(response.text)
# first entry is the query, the second one contains the suggestions
suggestions = suggestions[1]
# suggestions are given as tuples
# extract the string and skip the query if it occurs identically
suggestions = [
entry[0]
for entry in suggestions
if entry[0] != query and not song_utils.is_forbidden(musiq, entry[0])
]
return suggestions
def get_suggestions(self, request: WSGIRequest) -> JsonResponse:
"""Returns suggestions for a given query.
Combines online and offline suggestions."""
query = request.GET["term"]
suggest_playlist = request.GET["playlist"] == "true"
if self.musiq.base.settings.basic.new_music_only and not suggest_playlist:
return JsonResponse([], safe=False)
results = self._online_suggestions(query, suggest_playlist)
basic_settings = self.musiq.base.settings.basic
if suggest_playlist:
search_results = watson.search(query, models=(
ArchivedPlaylist, ))[:basic_settings.number_of_suggestions]
for playlist in search_results:
playlist_info = playlist.meta
archived_playlist = ArchivedPlaylist.objects.get(
id=playlist_info["id"])
result_dict: Dict[str, Union[str, int]] = {
"key": playlist_info["id"],
"value": playlist_info["title"],
"counter": playlist.object.counter,
"type":
song_utils.determine_playlist_type(archived_playlist),
}
results.append(result_dict)
else:
search_results = watson.search(
query,
models=(ArchivedSong, ))[:basic_settings.number_of_suggestions]
for search_result in search_results:
song_info = search_result.meta
if song_utils.is_forbidden(
self.musiq,
song_info["artist"]) or song_utils.is_forbidden(
self.musiq, song_info["title"]):
continue
try:
provider = SongProvider.create(
self.musiq, external_url=song_info["url"])
except NotImplementedError:
# For this song a provider is necessary that is not available
# e.g. the song was played before, but the provider was disabled
continue
cached = provider.check_cached()
# don't suggest local songs if they are not cached (=not at expected location)
if not cached and provider.type == "local":
continue
# don't suggest online songs when we don't have internet
if not self.musiq.base.settings.basic.has_internet and not cached:
continue
# don't suggest youtube songs if it was disabled
if (not self.musiq.base.settings.platforms.youtube_enabled
and provider.type == "youtube"):
continue
# don't suggest spotify songs if we are not logged in
if (not self.musiq.base.settings.platforms.spotify_enabled
and provider.type == "spotify"):
continue
# don't suggest soundcloud songs if we are not logged in
if (not self.musiq.base.settings.platforms.soundcloud_enabled
and provider.type == "soundcloud"):
continue
result_dict = {
"key":
song_info["id"],
"value":
song_utils.displayname(song_info["artist"],
song_info["title"]),
"counter":
search_result.object.counter,
"type":
provider.type,
}
results.append(result_dict)
return JsonResponse(results, safe=False)
def _offline_song_suggestions(query: str) -> List[SuggestionResult]:
results: List[SuggestionResult] = []
terms = query.split()
song_results: Iterable[Mapping[str, Any]]
if settings.DEBUG:
# Testing the whole table whether it contains any term is quite costly.
# Used for sqlite3 which does not have a similarity function.
matching_songs = ArchivedSong.objects.prefetch_related("queries")
for term in terms:
matching_songs = matching_songs.filter(
Q(title__icontains=term)
| Q(artist__icontains=term)
| Q(queries__query__icontains=term))
song_results = (
matching_songs.order_by("-counter")
# annotate with same values as in the postgres case to have a consistent interface
.annotate(
u_id=F("id"),
u_url=F("url"),
u_artist=F("artist"),
u_title=F("title"),
u_duration=F("duration"),
u_counter=F("counter"),
u_cached=F("cached"),
).values(*u_values_list).distinct()
[:storage.get("number_of_suggestions")])
# Perhaps this could be combined with the similarity search
# to improve usability with the right weighting.
# matching_songs = matching_songs.annotate(
# artist_similarity=Coalesce(TrigramWordSimilarity(query, "artist"), 0),
# title_similarity=Coalesce(TrigramWordSimilarity(query, "title"), 0),
# query_similarity=Coalesce(TrigramWordSimilarity(query, "queries__query"), 0),
# max_similarity=Greatest(
# "artist_similarity", "title_similarity", "query_similarity"
# ),
# )
# To combine, use union instead of | (or) in order to access the annotated values
# similar_songs = union(matching_songs)
else:
song_results = _postgres_song_results(query)
has_internet = redis.get("has_internet")
for song in song_results:
if song_utils.is_forbidden(
song["u_artist"]) or song_utils.is_forbidden(song["u_title"]):
continue
platform = song_utils.determine_url_type(song["u_url"])
# don't suggest online songs when we don't have internet
if not has_internet and not song["u_cached"]:
continue
if platform == "local":
# don't suggest local songs if they are not cached (=not at expected location)
if not song["u_cached"]:
continue
else:
# don't suggest songs if the respective platform is disabled
assert platform in ["youtube", "spotify", "soundcloud", "jamendo"]
if not storage.get(cast(PlatformEnabled, f"{platform}_enabled")):
continue
result_dict: SuggestionResult = {
"key": song["u_id"],
"value": song_utils.displayname(song["u_artist"], song["u_title"]),
"counter": song["u_counter"],
"type": platform,
"durationFormatted": song_utils.format_seconds(song["u_duration"]),
}
results.append(result_dict)
# mark suggestions whose displayname is identical
seen_values: Dict[str, int] = {}
for index, result in enumerate(results):
if result["value"] in seen_values:
result["confusable"] = True
results[seen_values[result["value"]]]["confusable"] = True
seen_values[result["value"]] = index
return results