def run(settings_dict, **kwargs):
"""
1. Update local music database.
2. Loop over each song without a local path.
3. Attempt to match that song with a local file.
4. Update the songs_dict if possible.
"""
database = kwargs["database"]
global_settings = kwargs["global_settings"]
component = kwargs["component"]
applet_id = kwargs["applet_id"]
songs_dict = kwargs["songs_dict"]
db = Database()
def update_database():
"""
Checks for modified directories since last scan.
Walks over all modified music files in the supplied directory and extracts their tags where supported.
Saves all tags to the database.
"""
# Create list of all found music files
songs = []
mtimes = []
songs_length = 0
log.info("Searching your local music directory for music files...")
for root, _, files in tqdm(os.walk(database["music_dir"]), desc="Searching for new songs"):
for item in files:
# Check extension is supported
_, ext = os.path.splitext(item)
ext = ext.lower()
if ext in extension_skiplist:
# Silently skip the file
continue
if ext not in supported_audio_extensions:
# Skip the file
log.debug(item)
log.debug(
f"Unsupported extension for local music file: {ext}")
continue
location = os.path.join(root, item)
# Check if file is already in database
mtime = db.item_exists(location)
new_mtime = math.floor(
os.path.getmtime(location)
)
if mtime == (new_mtime,):
# File already exists in database
continue
try:
song = local_tags.tags(location)
except NotImplementedError as e:
# Skip the file
log.debug(e)
log.debug(
f"Unsupported extension for local music file: {ext}")
continue
songs.append(song)
mtimes.append(new_mtime)
current_songs_length = len(mtimes)
if current_songs_length >= 100 + songs_length:
log.info(
f"Found {current_songs_length} songs not in the database.")
songs_length += 100
db.update_songs(songs, mtimes)
# 1. Update the database with any new songs added.
update_database()
preferred_order = ["isrc", "title", "artists", "album"]
total_count = 0
matched_count = 0
for i, playlist in enumerate(songs_dict):
for j, song in enumerate(playlist["songs"]):
if "location" in song.keys():
# Location already exists
continue
checked_songs = []
found = False
# Loop over available keys in order of preference
for key in [key for key in preferred_order if key in song]:
if key == "id":
continue
elif key == "artists":
resp = [db.get_song(key, value) for value in song[key]]
# Remove None from items
resp = [item for item in resp if item != None]
if resp:
# Flatten list response
resp = [item for sublist in resp for item in sublist]
else:
resp = db.get_song(key, song[key])
if resp:
for item in resp:
if item in checked_songs:
continue
score = fuzzymatch.similarity(song, item)
if score > float(database.get("fuzzy_ratio") or 90):
# Match found
songs_dict[i]["songs"][j]["location"] = item["location"]
found = True
break
checked_songs.extend(resp)
if found:
matched_count += 1
total_count += 1
break
if not found:
log.info(f"No local match was found for {song}")
total_count += 1
log.info(f"{matched_count} songs out of a total of {total_count} were matched with your local library, or already had a local path.")
return songs_dict
def search(self, track):
"""
Used to search the Spotify API for a song, supplied in standard songs_dict format.
Will attempt to match using fixed values (Spotify ID, ISRC) before moving onto fuzzy values.
@returns:
Spotify URI, confidence score
"""
cutoff_regex = [
"[([](feat|ft|featuring|original|prod).+?[)\]]",
"[ (\- )\-]+(feat|ft|featuring|original|prod).+?(?=[(\n])"
]
# 1. Spotify ID
try:
spotify_id = track["id"]["spotify"]
confidence = 100
return spotify_id, confidence
except KeyError:
# Spotify ID was not supplied
pass
# 2. Other fields
# Multiple searches are made as Spotify is more likely to return false negative (missing songs)
# than false positive, when specifying many query parameters.
queries = []
try:
# If ISRC exists, only use that query
queries.append(f"isrc:{track['isrc']}")
except KeyError:
# If no ISRC, add all additional queries
try:
title = re.sub(cutoff_regex[0],
"",
track['title'],
flags=re.IGNORECASE) + "\n"
title = re.sub(cutoff_regex[1],
" ",
title,
flags=re.IGNORECASE).strip()
except KeyError:
pass
try:
album = re.sub(cutoff_regex[0],
"",
track['album'],
flags=re.IGNORECASE) + "\n"
album = re.sub(cutoff_regex[1],
" ",
album,
flags=re.IGNORECASE).strip()
except KeyError:
pass
try:
queries.append(f"track:{title} album:{album}")
except NameError:
pass
try:
for artist in track["artists"]:
queries.append(f'track:"{title}" artist:"{artist}"')
except NameError:
pass
# try:
# queries.append(f"track:{title}")
# except NameError:
# pass
results_list = []
# Execute all queries
for query in queries:
results = self.request(self.sp.search, query)
# Convert to ultrasonics format and append to results_list
for item in results["tracks"]["items"]:
item = s.spotify_to_songs_dict(item)
if item not in results_list:
results_list.append(item)
if not results_list:
# No items were found
return "", 0
# Check results with fuzzy matching
confidence = 0
for item in results_list:
score = fuzzymatch.similarity(track, item)
if score > confidence:
matched_track = item
confidence = score
if confidence > 100:
break
spotify_id = matched_track['id']['spotify']
return spotify_id, confidence
def search(self, track):
"""
Used to search the Deezer API for a song, supplied in standard songs_dict format.
Will attempt to match using fixed values (Deezer ID, ISRC) before moving onto fuzzy values.
@returns:
Deezer ID, confidence score
"""
cutoff_regex = [
"[([](feat|ft|featuring|original|prod).+?[)\]]",
"[ (\- )\-]+(feat|ft|featuring|original|prod).+?(?=[(\n])"
]
# 1. Deezer ID
try:
deezer_id = track["id"]["deezer"]
confidence = 100
return deezer_id, confidence
except KeyError:
# Deezer ID was not supplied
pass
# 2. Other fields
# Multiple searches are made as Deezer is more likely to return false negative (missing songs)
# than false positive, when specifying many query parameters.
results_list = []
try:
# If ISRC exists, only use that query
url = f"https://api.deezer.com/2.0/track/isrc:{track['isrc']}"
resp = self.api(url)
if resp.get("error"):
# ISRC was not found in Deezer
raise KeyError
results_list.append(self.deezer_to_songs_dict(track=resp))
except KeyError:
# If no ISRC, try all additional queries
queries = []
try:
title = re.sub(cutoff_regex[0],
"",
track['title'],
flags=re.IGNORECASE) + "\n"
title = re.sub(cutoff_regex[1],
" ",
title,
flags=re.IGNORECASE).strip()
except KeyError:
pass
try:
album = re.sub(cutoff_regex[0],
"",
track['album'],
flags=re.IGNORECASE) + "\n"
album = re.sub(cutoff_regex[1],
" ",
album,
flags=re.IGNORECASE).strip()
except KeyError:
pass
try:
queries.append(f'track:"{title}" album:"{album}"')
except NameError:
pass
try:
for artist in track["artists"]:
queries.append(f'track:"{title}" artist:"{artist}"')
except NameError:
pass
# Execute all queries
url = "https://api.deezer.com/search"
for query in queries:
params = {"q": query, "limit": 20}
results = self.api(url, params=params)["data"]
# Convert to ultrasonics format and append to results_list
for result in results:
result = self.deezer_to_songs_dict(result=result)
if result not in results_list:
results_list.append(result)
if not results_list:
# No items were found
return "", 0
# Check results with fuzzy matching
confidence = 0
for item in results_list:
score = fuzzymatch.similarity(track, item)
if score > confidence:
matched_track = item
confidence = score
if confidence > 100:
break
deezer_id = matched_track["id"]["deezer"]
return deezer_id, confidence
def run(settings_dict, **kwargs):
"""
Runs the up_deezer plugin.
Important note: songs will only be appended to playlists if they are new!
No songs will be removed from existing playlists, nothing will be over-written.
This behaviour is different from some other plugins.
"""
database = kwargs["database"]
global_settings = kwargs["global_settings"]
component = kwargs["component"]
applet_id = kwargs["applet_id"]
songs_dict = kwargs["songs_dict"]
class Deezer:
"""
Class for interactions with Deezer through the Deezer api.
"""
def api(self, url, method="GET", params=None, data=None):
"""
Make a request to the Deezer API, with error handling.
@return: response JSON if successful
"""
if method == "GET":
r = requests.get(url, params=params)
if r.status_code == 4:
time.sleep(5)
r = requests.get(url, params=params)
elif method == "POST":
r = requests.post(url, data=data)
if r.status_code == 4:
time.sleep(5)
r = requests.post(url, data=data)
elif method == "DELETE":
r = requests.delete(url, params=params)
if r.status_code == 4:
time.sleep(5)
r = requests.post(url, data=data)
else:
raise Exception(f"Unknown api method: {method}")
if r.status_code != 200:
log.error(f"Unexpected status code: {r.status_code}")
log.error(r.text)
raise Exception("Unexpected status code")
try:
if r.json().get("error"):
log.error(f"An error was returned from the Deezer API.")
raise UserWarning(r.json()["error"])
except AttributeError:
# Returned data is not in JSON format
pass
return r.json()
def search(self, track):
"""
Used to search the Deezer API for a song, supplied in standard songs_dict format.
Will attempt to match using fixed values (Deezer ID, ISRC) before moving onto fuzzy values.
@returns:
Deezer ID, confidence score
"""
cutoff_regex = [
"[([](feat|ft|featuring|original|prod).+?[)\]]",
"[ (\- )\-]+(feat|ft|featuring|original|prod).+?(?=[(\n])"
]
# 1. Deezer ID
try:
deezer_id = track["id"]["deezer"]
confidence = 100
return deezer_id, confidence
except KeyError:
# Deezer ID was not supplied
pass
# 2. Other fields
# Multiple searches are made as Deezer is more likely to return false negative (missing songs)
# than false positive, when specifying many query parameters.
results_list = []
try:
# If ISRC exists, only use that query
url = f"https://api.deezer.com/2.0/track/isrc:{track['isrc']}"
resp = self.api(url)
if resp.get("error"):
# ISRC was not found in Deezer
raise KeyError
results_list.append(self.deezer_to_songs_dict(track=resp))
except KeyError:
# If no ISRC, try all additional queries
queries = []
try:
title = re.sub(cutoff_regex[0],
"",
track['title'],
flags=re.IGNORECASE) + "\n"
title = re.sub(cutoff_regex[1],
" ",
title,
flags=re.IGNORECASE).strip()
except KeyError:
pass
try:
album = re.sub(cutoff_regex[0],
"",
track['album'],
flags=re.IGNORECASE) + "\n"
album = re.sub(cutoff_regex[1],
" ",
album,
flags=re.IGNORECASE).strip()
except KeyError:
pass
try:
queries.append(f'track:"{title}" album:"{album}"')
except NameError:
pass
try:
for artist in track["artists"]:
queries.append(f'track:"{title}" artist:"{artist}"')
except NameError:
pass
# Execute all queries
url = "https://api.deezer.com/search"
for query in queries:
params = {"q": query, "limit": 20}
results = self.api(url, params=params)["data"]
# Convert to ultrasonics format and append to results_list
for result in results:
result = self.deezer_to_songs_dict(result=result)
if result not in results_list:
results_list.append(result)
if not results_list:
# No items were found
return "", 0
# Check results with fuzzy matching
confidence = 0
for item in results_list:
score = fuzzymatch.similarity(track, item)
if score > confidence:
matched_track = item
confidence = score
if confidence > 100:
break
deezer_id = matched_track["id"]["deezer"]
return deezer_id, confidence
def list_playlists(self):
"""
Wrapper for Deezer `list_playlists` which overcomes the request item limit.
"""
limit = 50
url = "https://api.deezer.com/user/me/playlists"
params = {"access_token": self.token, "limit": limit, "index": 0}
playlists = self.api(url, params=params)["data"]
playlist_count = len(playlists)
i = 1
# Get all playlists from the user
while playlist_count == limit:
params["index"] = limit * i
buffer = self.api(url, params=params)["data"]
playlists.extend(buffer)
playlist_count = len(buffer)
i += 1
log.info(f"Found {len(playlists)} playlist(s) on Deezer.")
return playlists
def playlist_tracks(self, playlist_id):
"""
Returns a list of tracks in a playlist.
"""
limit = 100
url = f"https://api.deezer.com/playlist/{playlist_id}/tracks"
params = {"access_token": self.token, "limit": limit, "index": 0}
tracks = self.api(url, params=params)["data"]
tracks_count = len(tracks)
i = 1
# Get all tracks from the playlist
while tracks_count == limit:
params["index"] = limit * i
buffer = self.api(url, params=params)["data"]
tracks.extend(buffer)
tracks_count = len(buffer)
i += 1
track_list = []
# Convert from Deezer API format to ultrasonics format
log.info("Converting tracks to ultrasonics format.")
for track in tqdm(tracks,
desc=f"Converting tracks in {playlist_id}"):
try:
track_list.append(self.deezer_to_songs_dict(result=track))
except UserWarning as e:
log.warning(
f"Unexpected response from Deezer for track {track}.")
log.warning(e)
return track_list
def remove_tracks_from_playlist(self, playlist_id, tracks):
"""
Removes all occurrences of `tracks` from the specified playlist.
"""
url = f"https://api.deezer.com/playlist/{playlist_id}/tracks"
params = {"access_token": self.token, "songs": ",".join(tracks)}
self.api(url, method="DELETE", params=params)
def deezer_to_songs_dict(self, track=None, result=None):
"""
Convert dictionary received from Deezer API to ultrasonics songs_dict format.
Songs can be converted from search result format (result), or direct song format (data)
Assumes title, artist(s), and id field are always present.
"""
if result:
# Get additional info about the track
url = f"https://api.deezer.com/track/{result['id']}"
track = self.api(url)
artists = [item["name"] for item in track["contributors"]]
try:
album = track["album"]["title"]
except KeyError:
album = None
try:
date = track["release_date"]
except KeyError:
date = None
try:
isrc = track["isrc"]
except KeyError:
isrc = None
item = {
"title": track["title"],
"artists": artists,
"album": album,
"date": date,
"isrc": isrc,
"id": {
"deezer": str(track["id"])
}
}
# Remove any empty fields
item = {k: v for k, v in item.items() if v}
return item
dz = Deezer()
dz.token = re.match("access_token=([\w]+)&", database["auth"]).groups()[0]
if component == "inputs":
# 1. Get a list of users playlists
playlists = dz.list_playlists()
songs_dict = []
for playlist in playlists:
item = {
"name": playlist["title"],
"id": {
"deezer": playlist["id"]
}
}
songs_dict.append(item)
# 2. Filter playlist titles
songs_dict = name_filter.filter(songs_dict, settings_dict["filter"])
# 3. Fetch songs from each playlist, build songs_dict
log.info("Building songs_dict for playlists...")
for i, playlist in tqdm(enumerate(songs_dict),
desc="Building songs_dict"):
tracks = dz.playlist_tracks(playlist["id"]["deezer"])
songs_dict[i]["songs"] = tracks
return songs_dict
else:
"Outputs mode"
# Get a list of current user playlists
current_playlists = dz.list_playlists()
for playlist in songs_dict:
# Check the playlist already exists in Deezer
playlist_id = ""
try:
if playlist["id"]["deezer"] in [
item["id"] for item in current_playlists
]:
playlist_id = playlist["id"]["deezer"]
except KeyError:
if playlist["name"] in [
item["title"] for item in current_playlists
]:
playlist_id = [
item["id"] for item in current_playlists
if item["title"] == playlist["name"]
][0]
if playlist_id:
log.info(
f"Playlist {playlist['name']} already exists, updating that one."
)
else:
# Playlist must be created
log.info(
f"Playlist {playlist['name']} does not exist, creating it now..."
)
url = "https://api.deezer.com/user/me/playlists"
data = {"access_token": dz.token, "title": playlist["name"]}
playlist_id = dz.api(url, method="POST", data=data)["id"]
public = "true" if database[
"created_playlists"] == "Public" else "false"
description = "Created automatically by ultrasonics with 💖"
url = f"https://api.deezer.com/playlist/{playlist_id}"
data = {
"access_token": dz.token,
"public": public,
"description": description
}
response = dz.api(url, method="POST", data=data)
if not response is True:
raise Exception(
f"Unexpected response while updating playlist: {response}"
)
existing_tracks = []
existing_ids = []
# Get all tracks already in the playlist
if "existing_tracks" not in vars():
existing_tracks = dz.playlist_tracks(playlist_id)
existing_ids = [
str(item["id"]["deezer"]) for item in existing_tracks
]
# Add songs which don't already exist in the playlist
new_ids = []
duplicate_ids = []
log.info("Searching for matching songs in Deezer.")
for song in tqdm(
playlist["songs"],
desc=f"Searching Deezer for songs from {playlist['name']}"
):
# First check for fuzzy duplicate without Deezer api search
duplicate = False
for item in existing_tracks:
score = fuzzymatch.similarity(song, item)
if score > float(database.get("fuzzy_ratio") or 90):
# Duplicate was found
duplicate_ids.append(item['id']['deezer'])
duplicate = True
break
if duplicate:
continue
try:
deezer_id, confidence = dz.search(song)
except UserWarning:
# Likely no data was returned
log.warning(
f"No data was returned when searching for song: {song}"
)
continue
if deezer_id in existing_ids:
duplicate_ids.append(deezer_id)
if confidence > float(database.get("fuzzy_ratio") or 90):
new_ids.append(str(deezer_id))
else:
log.debug(
f"Could not find song {song['title']} in Deezer; will not add to playlist."
)
if settings_dict["existing_playlists"] == "Update":
# Remove any songs which aren't in `uris` from the playlist
remove_ids = [
deezer_id for deezer_id in existing_ids
if deezer_id not in new_ids + duplicate_ids
]
dz.remove_tracks_from_playlist(playlist_id, remove_ids)
# Add tracks to playlist in batches of 100
url = f"https://api.deezer.com/playlist/{playlist_id}/tracks"
data = {"access_token": dz.token}
while len(new_ids) > 100:
data["songs"] = ",".join(new_ids[0:100])
dz.api(url, method="POST", data=data)
new_ids = new_ids[100:]
# Add all remaining tracks
if new_ids:
data["songs"] = ",".join(new_ids)
dz.api(url, method="POST", data=data)
def run(settings_dict, **kwargs):
"""
Runs the up_spotify plugin.
Important note: songs will only be appended to playlists if they are new!
No songs will be removed from existing playlists, nothing will be over-written.
This behaviour is different from some other plugins.
"""
database = kwargs["database"]
global_settings = kwargs["global_settings"]
component = kwargs["component"]
applet_id = kwargs["applet_id"]
songs_dict = kwargs["songs_dict"]
class Spotify:
"""
Class for interactions with Spotify through the Spotipy api.
"""
def __init__(self):
self.cache_file = os.path.join(
_ultrasonics["config_dir"], "up_spotify", "up_spotify.bz2")
log.info(f"Credentials will be cached in: {self.cache_file}")
# Create the containing folder if it doesn't already exist
try:
os.mkdir(os.path.dirname(self.cache_file))
except FileExistsError:
# Folder already exists
pass
def token_get(self, force=False):
"""
Updates the global access_token variable, in the following order of preference:
1. A locally saved access token.
2. Renews token using the database refresh_token.
If #2, the access token is saved to a .cache file.
"""
log.debug("Fetching your Spotify token")
if os.path.isfile(self.cache_file) and not force:
with bz2.BZ2File(self.cache_file, "r") as f:
raw = json.loads(pickle.load(f))
token = raw["access_token"]
# Checks that the cached token is valid
if self.token_validate(token):
log.debug("Returning cached token")
return token
# In all other cases, renew the token
log.debug("Returning renewed token")
return self.token_renew()
def token_validate(self, token):
"""
Checks if an auth token is valid, by making a request to the Spotify api.
"""
url = f"https://api.spotify.com/v1/search"
params = {
"q": "Flume",
"type": "artist"
}
headers = {
"Authorization": f"Bearer {token}"
}
resp = requests.get(url, headers=headers, params=params)
if resp.status_code == 200:
log.debug("Token is valid.")
return True
elif resp.status_code == 401:
log.debug("Token is not valid.")
return False
else:
log.error(f"Unexpected status code: {resp.status_code}")
raise Exception(resp.text)
def token_renew(self):
"""
Using refresh_token, requests and saves a new access token.
"""
url = urljoin(self.api_url, "spotify/auth/renew")
data = {
"refresh_token": self.refresh_token,
"ultrasonics_auth_hash": api_key.get_hash(True)
}
log.info(
"Requesting a new Spotify token, this may take a few seconds...")
# Request with a long timeout to account for free Heroku start-up 😉
resp = requests.post(url, data=data, timeout=60)
if resp.status_code == 200:
token = resp.json()["access_token"]
log.debug(
f"Spotify renew data: {resp.text.replace(token, '***************')}")
with bz2.BZ2File(self.cache_file, "w") as f:
pickle.dump(resp.text, f)
return token
else:
log.error(resp.text)
raise Exception(
f"The response `when renewing Spotify token was unexpected: {resp.status_code}")
def request(self, sp_func, *args, **kwargs):
"""
Used to call a spotipy function, with automatic catching and renewing on access token errors.
"""
errors = 0
while errors <= 1:
try:
return sp_func(*args, **kwargs)
except spotipy.exceptions.SpotifyException as e:
# Renew token
log.error(e)
self.sp = spotipy.Spotify(auth=self.token_get(force=True))
errors += 1
continue
log.error(
"An error occurred while trying to contact the Spotify api.")
raise Exception(e)
def search(self, track):
"""
Used to search the Spotify API for a song, supplied in standard songs_dict format.
Will attempt to match using fixed values (Spotify ID, ISRC) before moving onto fuzzy values.
@returns:
Spotify URI, confidence score
"""
cutoff_regex = [
"[([](feat|ft|featuring|original|prod).+?[)\]]",
"[ (\- )\-]+(feat|ft|featuring|original|prod).+?(?=[(\n])"
]
# 1. Spotify ID
try:
spotify_id = track["id"]["spotify"]
spotify_uri = f"spotify:track:{spotify_id}"
confidence = 100
return spotify_uri, confidence
except KeyError:
# Spotify ID was not supplied
pass
# 2. Other fields
# Multiple searches are made as Spotify is more likely to return false negative (missing songs)
# than false positive, when specifying many query parameters.
queries = []
try:
# If ISRC exists, only use that query
queries.append(f"isrc:{track['isrc']}")
except KeyError:
# If no ISRC, add all additional queries
try:
title = re.sub(cutoff_regex[0], "",
track['title'], flags=re.IGNORECASE) + "\n"
title = re.sub(cutoff_regex[1], " ",
title, flags=re.IGNORECASE).strip()
except KeyError:
pass
try:
album = re.sub(cutoff_regex[0], "",
track['album'], flags=re.IGNORECASE) + "\n"
album = re.sub(cutoff_regex[1], " ",
album, flags=re.IGNORECASE).strip()
except KeyError:
pass
try:
queries.append(f"track:{title} album:{album}")
except NameError:
pass
try:
for artist in track["artists"]:
queries.append(
f'track:"{title}" artist:"{artist}"')
except NameError:
pass
# try:
# queries.append(f"track:{title}")
# except NameError:
# pass
results_list = []
# Execute all queries
for query in queries:
results = self.request(self.sp.search, query)
# Convert to ultrasonics format and append to results_list
for item in results["tracks"]["items"]:
item = s.spotify_to_songs_dict(item)
if item not in results_list:
results_list.append(item)
if not results_list:
# No items were found
return "", 0
# Check results with fuzzy matching
confidence = 0
for item in results_list:
score = fuzzymatch.similarity(track, item)
if score > confidence:
matched_track = item
confidence = score
if confidence > 100:
break
spotify_uri = f"spotify:track:{matched_track['id']['spotify']}"
return spotify_uri, confidence
def current_user_playlists(self):
"""
Wrapper for Spotify `current_user_playlists` which overcomes the request item limit.
"""
limit = 50
playlists = self.request(self.sp.current_user_playlists,
limit=limit, offset=0)["items"]
playlist_count = len(playlists)
i = 1
# Get all playlists from the user
while playlist_count == limit:
buffer = self.request(self.sp.current_user_playlists,
limit=limit, offset=limit * i)["items"]
playlists.extend(buffer)
playlist_count = len(buffer)
i += 1
log.info(f"Found {len(playlists)} playlist(s) on Spotify.")
return playlists
def current_user_saved_tracks(self, page=0):
"""
Wrapper for Spotipy `current_user_saved_tracks`, which allows page selection to get earlier tracks.
"""
limit = 20
offset = limit * page
tracks = self.request(self.sp.current_user_saved_tracks,
limit=limit, offset=offset)["items"]
spotify_ids = [item["track"]["id"] for item in tracks]
return spotify_ids, tracks
def playlist_tracks(self, playlist_id):
"""
Wrapper for Spotipy `playlist_tracks` which overcomes the request item limit.
"""
limit = 100
fields = "items(track(album(name,release_date),artists,id,name,track_number,external_ids))"
tracks = self.request(self.sp.playlist_tracks, playlist_id,
limit=limit, offset=0, fields=fields)
tracks_count = len(tracks)
i = 1
# Get all tracks from the playlist
while tracks_count == limit:
buffer = self.request(self.sp.playlist_tracks, playlist_id,
limit=limit, offset=limit * i, **kwargs)
tracks.extend(buffer)
tracks_count = len(buffer)
i += 1
tracks = tracks["items"]
track_list = []
# Convert from Spotify API format to ultrasonics format
log.info("Converting tracks to ultrasonics format.")
for track in tqdm([track["track"] for track in tracks], desc=f"Converting tracks in {playlist_id}"):
track_list.append(s.spotify_to_songs_dict(track))
return track_list
def user_playlist_remove_all_occurrences_of_tracks(self, playlist_id, tracks):
"""
Wrapper for the spotipy function of the same name.
Removes all `tracks` from the specified playlist.
"""
self.request(
self.sp.user_playlist_remove_all_occurrences_of_tracks, self.user_id, playlist_id, tracks)
def spotify_to_songs_dict(self, track):
"""
Convert dictionary received from Spotify API to ultrasonics songs_dict format.
Assumes title, artist(s), and id field are always present.
"""
artists = [artist["name"] for artist in track["artists"]]
try:
album = track["album"]["name"]
except KeyError:
album = None
try:
date = track["album"]["release_date"]
except KeyError:
date = None
try:
isrc = track["external_ids"]["isrc"]
except KeyError:
isrc = None
item = {
"title": track["name"],
"artists": artists,
"album": album,
"date": date,
"isrc": isrc
}
if track["id"]:
item["id"] = {"spotify": str(track["id"])}
else:
log.debug(f"Invalid spotify id for song: {track['name']}")
# Remove any empty fields
item = {k: v for k, v in item.items() if v}
return item
def user_id(self):
"""
Get and return the current user's user ID.
"""
user_info = self.request(self.sp.current_user)
self.user_id = user_info["id"]
class Database:
"""
Class for interactions with the up_spotify database.
Currently used for storing info about saved songs.
"""
def __init__(self):
# Create database if required
self.saved_songs_db = os.path.join(
_ultrasonics["config_dir"], "up_spotify", "saved_songs.db")
with sqlite3.connect(self.saved_songs_db) as conn:
cursor = conn.cursor()
# Create saved songs table if needed
query = "CREATE TABLE IF NOT EXISTS saved_songs (applet_id TEXT, spotify_id TEXT)"
cursor.execute(query)
# Create lastrun table if needed
query = "CREATE TABLE IF NOT EXISTS lastrun (applet_id TEXT PRIMARY KEY, time INTEGER)"
cursor.execute(query)
conn.commit()
def lastrun_get(self):
"""
Gets the last run time for saved songs mode.
"""
with sqlite3.connect(self.saved_songs_db) as conn:
cursor = conn.cursor()
query = "SELECT time FROM lastrun WHERE applet_id = ?"
cursor.execute(query, (applet_id,))
rows = cursor.fetchone()
return None if not rows else rows[0]
def lastrun_set(self):
"""
Updates the last run time for saved songs mode.
"""
with sqlite3.connect(self.saved_songs_db) as conn:
cursor = conn.cursor()
query = "REPLACE INTO lastrun (time, applet_id) VALUES (?, ?)"
cursor.execute(query, (int(time.time()), applet_id))
def saved_songs_contains(self, spotify_id):
"""
Checks if the input `spotify_id` is present in the saved songs database.
"""
with sqlite3.connect(self.saved_songs_db) as conn:
cursor = conn.cursor()
query = "SELECT EXISTS(SELECT * FROM saved_songs WHERE applet_id = ? AND spotify_id = ?)"
cursor.execute(query, (applet_id, spotify_id))
rows = cursor.fetchone()
return rows[0]
def saved_songs_add(self, spotify_ids):
"""
Adds all songs in a list of `spotify_ids` to the saved songs database for this applet.
"""
with sqlite3.connect(self.saved_songs_db) as conn:
cursor = conn.cursor()
# Add saved songs to database
query = "INSERT INTO saved_songs (applet_id, spotify_id) VALUES (?, ?)"
values = [(applet_id, spotify_id)
for spotify_id in spotify_ids]
cursor.executemany(query, values)
conn.commit()
s = Spotify()
db = Database()
s.api_url = global_settings["api_url"]
auth = json.loads(database["auth"])
s.refresh_token = auth["refresh_token"]
s.sp = spotipy.Spotify(auth=s.token_get(), requests_timeout=60)
if component == "inputs":
if settings_dict["mode"] == "playlists":
# Playlists mode
# 1. Get a list of users playlists
playlists = s.current_user_playlists()
songs_dict = []
for playlist in playlists:
item = {
"name": playlist["name"],
"id": {
"spotify": playlist["id"]
}
}
songs_dict.append(item)
# 2. Filter playlist titles
songs_dict = name_filter.filter(
songs_dict, settings_dict["filter"])
# 3. Fetch songs from each playlist, build songs_dict
log.info("Building songs_dict for playlists...")
for i, playlist in tqdm(enumerate(songs_dict)):
tracks = s.playlist_tracks(playlist["id"]["spotify"])
songs_dict[i]["songs"] = tracks
return songs_dict
elif settings_dict["mode"] == "saved":
# Saved songs mode
if db.lastrun_get():
# Update songs
songs = []
reached_limit = False
page = 0
# Loop until a known saved song is found
while not reached_limit:
spotify_ids, tracks = s.current_user_saved_tracks(
page=page)
for spotify_id, track in zip(spotify_ids, tracks):
if db.saved_songs_contains(spotify_id):
reached_limit = True
break
else:
songs.append(
s.spotify_to_songs_dict(track["track"]))
page += 1
if not songs:
log.info(
"No new saved songs were found. Exiting this applet.")
raise Exception(
"No new saved songs found on this applet run.")
songs_dict = [
{
"name": settings_dict["playlist_title"] or "Spotify Saved Songs",
"id": {},
"songs": songs
}
]
return songs_dict
else:
log.info(
"This is the first time this applet plugin has run in saved songs mode.")
log.info(
"This first run will be used to get the current state of saved songs, but will not pass any songs in songs_dict.")
# 1. Get some saved songs
spotify_ids, _ = s.current_user_saved_tracks()
# 2. Update database with saved songs
db.saved_songs_add(spotify_ids)
# 3. Update lastrun
db.lastrun_set()
raise Exception(
"Initial run of this plugin will not return a songs_dict. Database is now updated. Next run will continue as normal.")
else:
"Outputs mode"
# Set the user_id variable
s.user_id()
# Get a list of current user playlists
current_playlists = s.current_user_playlists()
for playlist in songs_dict:
# Check the playlist already exists in Spotify
playlist_id = ""
try:
if playlist["id"]["spotify"] in [item["id"] for item in current_playlists]:
playlist_id = playlist["id"]["spotify"]
except KeyError:
if playlist["name"] in [item["name"] for item in current_playlists]:
playlist_id = [
item["id"] for item in current_playlists if item["name"] == playlist["name"]][0]
if playlist_id:
log.info(
f"Playlist {playlist['name']} already exists, updating that one.")
else:
log.info(
f"Playlist {playlist['name']} does not exist, creating it now...")
# Playlist must be created
public = database["created_playlists"] == "Public"
description = "Created automatically by ultrasonics with 💖"
response = s.request(s.sp.user_playlist_create, s.user_id,
playlist["name"], public=public, description=description)
playlist_id = response["id"]
existing_tracks = []
existing_uris = []
# Get all tracks already in the playlist
if "existing_tracks" not in vars():
existing_tracks = s.playlist_tracks(playlist_id)
existing_uris = [
f"spotify:track:{item['id']['spotify']}" for item in existing_tracks]
# Add songs which don't already exist in the playlist
uris = []
duplicate_uris = []
log.info("Searching for matching songs in Spotify.")
for song in tqdm(playlist["songs"], desc=f"Searching Spotify for songs from {playlist['name']}"):
# First check for fuzzy duplicate without Spotify api search
duplicate = False
for item in existing_tracks:
score = fuzzymatch.similarity(song, item)
if score > float(database.get("fuzzy_ratio") or 90):
# Duplicate was found
duplicate_uris.append(
f"spotify:track:{item['id']['spotify']}")
duplicate = True
break
if duplicate:
continue
uri, confidence = s.search(song)
if uri in existing_uris:
duplicate_uris.append(uri)
if confidence > float(database.get("fuzzy_ratio") or 90):
uris.append(uri)
else:
log.debug(
f"Could not find song {song['title']} in Spotify; will not add to playlist.")
if settings_dict["existing_playlists"] == "Update":
# Remove any songs which aren't in `uris` from the playlist
remove_uris = [
uri for uri in existing_uris if uri not in uris + duplicate_uris]
s.user_playlist_remove_all_occurrences_of_tracks(
playlist_id, remove_uris)
# Add tracks to playlist in batches of 100
while len(uris) > 100:
s.request(s.sp.user_playlist_add_tracks, s.user_id,
playlist_id, uris[0:100])
uris = uris[100:]
# Add all remaining tracks
if uris:
s.request(s.sp.user_playlist_add_tracks, s.user_id,
playlist_id, uris)