def _get_itunes_album_artwork_uri(self, artist, album, cancellable=None):
"""
Get album artwork using Itunes
@param artist as str
@param album as str
@param cancellable as Gio.Cancellable
@return uri as str
@tread safe
"""
if not get_network_available("ITUNES"):
return []
try:
album_formated = GLib.uri_escape_string(
album, None, True).replace(" ", "+")
uri = "https://itunes.apple.com/search" +\
"?entity=album&term=%s" % album_formated
(status, data) = App().task_helper.load_uri_content_sync(
uri, cancellable)
if status:
decode = json.loads(data.decode("utf-8"))
for item in decode["results"]:
if noaccents(item["artistName"].lower()) ==\
noaccents(artist.lower()):
uri = item["artworkUrl60"].replace("60x60",
"1024x1024")
return [uri]
except Exception as e:
Logger.warning("%s %s %s", e, artist, album)
Logger.warning(
"DownloaderArt::_get_album_art_itunes_uri: %s", data)
return []
def _get_fanarttv_album_artwork_uri(self, artist, album, cancellable=None):
"""
Get album artwork using FanartTV
@param artist as str
@param album as str
@param cancellable as Gio.Cancellable
@return uri as str
@thread safe
"""
if not get_network_available("FANARTTV"):
return []
uris = []
try:
search = "%s %s" % (artist, album)
mbid = self.__get_musicbrainz_mbid("album", search, cancellable)
if mbid is None:
return []
uri = "http://webservice.fanart.tv/v3/music/albums/%s?api_key=%s"
(status, data) = App().task_helper.load_uri_content_sync(
uri % (mbid, FANARTTV_ID), cancellable)
if status:
decode = json.loads(data.decode("utf-8"))
for cover in decode["albums"][mbid]["albumcover"]:
uris.append(cover["url"])
except Exception as e:
Logger.warning("%s %s %s", e, artist, album)
Logger.warning(
"DownloaderArt::_get_fanarttv_album_artwork_uri: %s", data)
return uris
def _get_deezer_album_artwork_uri(self, artist, album, cancellable=None):
"""
Get album artwork using Deezer
@param artist as str
@param album as str
@param cancellable as Gio.Cancellable
@return uri as str
@tread safe
"""
if not get_network_available("DEEZER"):
return []
try:
album_formated = GLib.uri_escape_string(album, None, True)
uri = "https://api.deezer.com/search/album/?" +\
"q=%s&output=json" % album_formated
(status, data) = App().task_helper.load_uri_content_sync(
uri, cancellable)
if status:
decode = json.loads(data.decode("utf-8"))
uri = None
for item in decode["data"]:
if noaccents(item["artist"]["name"].lower()) ==\
noaccents(artist.lower()):
uri = item["cover_xl"]
return [uri]
except Exception as e:
Logger.warning("%s %s %s", e, artist, album)
Logger.warning("DownloaderArt::__get_deezer_album_artwork_uri: %s",
data)
return []
def _get_spotify_artist_artwork_uri(self, artist, cancellable=None):
"""
Get artist artwork using Spotify
@param artist as str
@param cancellable as Gio.Cancellable
@return uri as str
@tread safe
"""
if not get_network_available("SPOTIFY"):
return []
try:
artist_formated = GLib.uri_escape_string(
artist, None, True).replace(" ", "+")
uri = "https://api.spotify.com/v1/search?q=%s" % artist_formated +\
"&type=artist"
token = App().ws_director.token_ws.get_token("SPOTIFY",
cancellable)
bearer = "Bearer %s" % token
headers = [("Authorization", bearer)]
(status,
data) = App().task_helper.load_uri_content_sync_with_headers(
uri, headers, cancellable)
if status:
uri = None
decode = json.loads(data.decode("utf-8"))
for item in decode["artists"]["items"]:
if noaccents(item["name"].lower()) ==\
noaccents(artist.lower()):
uri = item["images"][0]["url"]
return [uri]
except Exception as e:
Logger.warning("%s %s", e, artist)
Logger.warning(
"DownloaderArt::_get_spotify_artist_artwork_uri(): %s", data)
return []
def _get_audiodb_album_artwork_uri(self, artist, album, cancellable=None):
"""
Get album artwork using AudioDB
@param artist as str
@param album as str
@param cancellable as Gio.Cancellable
@return uri as str
@thread safe
"""
if not get_network_available("AUDIODB"):
return []
try:
album = GLib.uri_escape_string(album, None, True)
artist = GLib.uri_escape_string(artist, None, True)
uri = "https://theaudiodb.com/api/v1/json/"
uri += "%s/searchalbum.php?s=%s&a=%s" % (AUDIODB_CLIENT_ID,
artist,
album)
(status, data) = App().task_helper.load_uri_content_sync(
uri, cancellable)
if status:
decode = json.loads(data.decode("utf-8"))
if decode["album"]:
for item in decode["album"]:
uri = item["strAlbumThumb"]
return [uri]
except Exception as e:
Logger.warning("%s %s %s", e, artist, album)
Logger.warning(
"DownloaderArt::_get_audiodb_album_artwork_uri: %s", data)
return []
def decode_lyrics(bytes):
try:
lyrics = b""
if bytes[0:4] == b"TXXX":
if bytes[13:24] == b"L\x00y\x00r\x00i\x00c\x00s":
lyrics = bytes[29:].replace(b"\x00", b"")
else:
return None
elif bytes[0:4] == b"USLT":
# This code sucks, if someone know how to handle this
# UTF8
lyrics = bytes.split(b"\x00")[-1]
# UTF-16
if not lyrics:
lyrics = bytes.split(
b"\xff\xfe")[2].replace(b"\x00", b"")
else:
return None
for encoding in ENCODING:
try:
return lyrics.decode(encoding)
except Exception as e:
Logger.warning("TagReader::get_lyrics(ENCODING): %s",
e)
except Exception as e:
Logger.warning("TagReader::get_lyrics(): %s", e)
return None
def __remove_old_tracks(self, uris, scan_type):
"""
Remove non existent tracks from DB
@param scan_type as ScanType
"""
if scan_type != ScanType.EXTERNAL and self.__thread is not None:
# We need to check files are always in collections
if scan_type == ScanType.FULL:
collections = App().settings.get_music_uris()
else:
collections = None
for uri in uris:
# Handle a stop request
if self.__thread is None:
raise Exception("cancelled")
in_collection = True
if collections is not None:
in_collection = False
for collection in collections:
if collection in uri:
in_collection = True
break
f = Gio.File.new_for_uri(uri)
if not in_collection:
Logger.warning(
"Removed, not in collection anymore: %s -> %s", uri,
collections)
self.del_from_db(uri, True)
elif not f.query_exists():
Logger.warning("Removed, file has been deleted: %s", uri)
self.del_from_db(uri, True)
def search_charts(self, cancellable):
"""
Add charts to DB
@param cancellable as Gio.Cancellable
"""
Logger.info("Get charts with Deezer")
try:
album_ids = []
uri = "https://api.deezer.com/chart/0/albums?limit=30"
(status, data) = App().task_helper.load_uri_content_sync(
uri, cancellable)
if status:
decode = json.loads(data.decode("utf-8"))
for album in decode["data"]:
album_ids.append(album["id"])
for album_id in album_ids:
if cancellable.is_cancelled():
raise Exception("Cancelled")
payload = DeezerWebHelper.get_album_payload(
self, album_id, cancellable)
if payload is None:
continue
lollypop_payload = DeezerWebHelper.lollypop_album_payload(
self, payload)
self.save_album_payload_to_db(lollypop_payload,
StorageType.DEEZER_CHARTS,
True,
cancellable)
except Exception as e:
Logger.warning(
"DeezerCollectionWebService::search_charts(): %s", e)
def __on_load_uri_content(self, uri, loaded, content, api, uris):
"""
Add loaded pixbuf
@param uri as str
@param loaded as bool
@param content as bytes
@param uris as [str]
@param api as str
@param last as bool
"""
try:
if loaded:
self.__add_pixbuf(content, api)
if uris:
(uri, api) = uris.pop(0)
App().task_helper.load_uri_content(uri,
self._cancellable,
self.__on_load_uri_content,
api,
uris)
else:
self.__loaders -= 1
except Exception as e:
self.__loaders -= 1
Logger.warning(
"ArtworkSearchWidget::__on_load_uri_content(): %s", e)
if self.__loaders == 0:
self.__spinner.stop()
def __on_get_youtube_id(self, uri, status, content, track, cancellable,
methods):
"""
Get youtube id or run another method if not found
@param uri as str
@param status as bool
@param content as bytes
@param track as Track
@param cancellable as Gio.Cancellable
@param methods as [function]
"""
try:
youtube_id = None
if status:
decode = json.loads(content.decode("utf-8"))
dic = {}
best = self.__BAD_SCORE
for i in decode["items"]:
score = get_page_score(i["snippet"]["title"], track.name,
track.artists[0], track.album.name)
if score == -1 or score == best:
continue
elif score < best:
best = score
dic[score] = i["id"]["videoId"]
# Return url from first dic item
if best != self.__BAD_SCORE:
youtube_id = dic[best]
except:
Logger.warning("YouTubeHelper::__on_get_youtube_id(): %s", content)
self.__emit_uri_loaded(youtube_id, track, cancellable, methods)
def load_tracks(self, album, cancellable):
"""
Load tracks for album
@param album as Album
@param cancellable as Gio.Cancellable
"""
try:
spotify_id = album.uri.replace("sp:", "")
token = App().ws_director.token_ws.get_token(
"SPOTIFY", cancellable)
uri = "https://api.spotify.com/v1/albums/%s" % spotify_id
token = App().ws_director.token_ws.get_token(
"SPOTIFY", cancellable)
bearer = "Bearer %s" % token
headers = [("Authorization", bearer)]
(status,
data) = App().task_helper.load_uri_content_sync_with_headers(
uri, headers, cancellable)
if status:
decode = json.loads(data.decode("utf-8"))
# We want to share the same item as lp_album_id may change
album_item = album.collection_item
for track in decode["tracks"]["items"]:
lollypop_payload = self.lollypop_track_payload(track)
self.save_track_payload_to_db(lollypop_payload, album_item,
album.storage_type, False,
cancellable)
except Exception as e:
Logger.warning("SpotifySearch::load_tracks(): %s, %s", e, data)
def get_today_album():
"""
Get today album
@return Album/None
"""
current_date = GLib.DateTime.new_now_local().get_day_of_year()
(date, album_id) = (0, None)
try:
(date,
album_id) = load(open(LOLLYPOP_DATA_PATH + "/today.bin", "rb"))
if App().albums.get_storage_type(album_id) == StorageType.NONE:
date = 0
except Exception as e:
Logger.warning("TodayBannerWidget::__get_today_album(): %s", e)
try:
if date != current_date:
storage_type = get_default_storage_type()
album_id = App().albums.get_randoms(storage_type, None, False,
1)[0]
dump((current_date, album_id),
open(LOLLYPOP_DATA_PATH + "/today.bin", "wb"))
return Album(album_id)
except Exception as e:
Logger.error("TodayBannerWidget::__get_today_album(): %s", e)
return None
def __populate_db(self):
"""
Populate DB in a background task
"""
try:
Logger.info("Collection download started")
self.__is_running = True
self.__cancellable = Gio.Cancellable()
storage_types = []
mask = App().settings.get_value("suggestions-mask").get_int32()
# Check if storage type needs to be updated
# Check if albums newer than a week are enough
timestamp = time() - 604800
for storage_type in self.__STORAGE_TYPES:
if not mask & storage_type:
continue
newer_albums = App().albums.get_newer_for_storage_type(
storage_type, timestamp)
if len(newer_albums) < self.MIN_ITEMS_PER_STORAGE_TYPE:
storage_types.append(storage_type)
# Update needed storage types
if storage_types:
for storage_type in storage_types:
if self.__cancellable.is_cancelled():
raise Exception("cancelled")
self.__METHODS[storage_type](self, self.__cancellable)
self.clean_old_albums(storage_types)
App().artists.update_featuring()
except Exception as e:
Logger.warning("CollectionWebService::__populate_db(): %s", e)
self.__is_running = False
Logger.info("Collection download finished")
def __add_sync_action(self, name):
"""
Add sync action
@param name as str
@param status as bool
"""
def on_get_synced(synced, sync_action):
sync_action.set_state(GLib.Variant.new_boolean(synced))
synced = False
devices = list(App().settings.get_value("devices"))
action_name = "sync_%s" % name
encoded = sha256(action_name.encode("utf-8")).hexdigest()
sync_action = Gio.SimpleAction.new_stateful(
encoded, None, GLib.Variant.new_boolean(synced))
App().add_action(sync_action)
try:
if name == self.__all_devices:
index = 0
else:
index = devices.index(name) + 1
App().task_helper.run(self._get_synced,
index,
callback=(on_get_synced, sync_action))
except Exception as e:
Logger.warning("SyncMenu::__add_sync_action(): %s", e)
if name != self.__all_devices:
self.__actions.append(sync_action)
sync_action.connect("change-state", self.__on_sync_action_change_state,
name)
self.append(name, "app.%s" % encoded)
return synced
def search(self, search, cancellable):
"""
Get albums related to search
We need a thread because we are going to populate DB
@param search as str
@param cancellable as Gio.Cancellable
"""
try:
while self.wait_for_token():
if cancellable.is_cancelled():
raise Exception("cancelled")
sleep(1)
token = "Bearer %s" % self.__token
helper = TaskHelper()
helper.add_header("Authorization", token)
uri = "https://api.spotify.com/v1/search?"
uri += "q=%s&type=album,track" % search
(status, data) = helper.load_uri_content_sync(uri, cancellable)
if status:
decode = json.loads(data.decode("utf-8"))
album_ids = []
self.__create_albums_from_album_payload(
decode["albums"]["items"],
album_ids,
cancellable)
self.__create_albums_from_tracks_payload(
decode["tracks"]["items"],
album_ids,
cancellable)
except Exception as e:
Logger.warning("SpotifyHelper::search(): %s", e)
# Do not emit search-finished on cancel
if str(e) == "cancelled":
return
GLib.idle_add(self.emit, "search-finished")
def __on_load_uri_content(self, source, result, msg, headers, callback,
cancellable, uri, *args):
"""
Get stream and start reading from it
@param source as Soup.Session
@param result as Gio.AsyncResult
@param msg as Soup.Message
@param headers as []
@param cancellable as Gio.Cancellable
@param callback as a function
@param uri as str
"""
try:
response_headers = msg.get_property("response-headers")
wait = self.__handle_ratelimit(response_headers, uri)
if wait is None:
stream = source.send_finish(result)
# We use a bytearray here as seems that bytes += is really slow
stream.read_bytes_async(4096, GLib.PRIORITY_LOW, cancellable,
self.__on_read_bytes_async,
bytearray(0), cancellable, callback,
uri, *args)
else:
parsed = urlparse(uri)
self.__ratelimit[parsed.netloc] = wait
retries = self.__get_retries_for_uri(uri)
if retries < 5:
self.__retries[uri] += 1
self.load_uri_content_sync_with_headers(
uri, headers, callback, *args)
else:
del self.__retries[uri]
except Exception as e:
Logger.warning("TaskHelper::__on_soup_msg_finished(): %s" % e)
callback(uri, False, b"", *args)
def get(self, search, storage_type, cancellable):
"""
Get albums for search
We need a thread because we are going to populate DB
@param search as str
@param storage_type as StorageType
@param cancellable as Gio.Cancellable
"""
if not get_network_available("DEEZER"):
emit_signal(self, "finished")
return
try:
uri = "https://api.deezer.com/search/album?q=%s" %\
search
(status, data) = App().task_helper.load_uri_content_sync(
uri, cancellable)
if status:
decode = json.loads(data.decode("utf-8"))
for albums in decode["data"]:
payload = self.lollypop_album_payload(albums)
self.save_album_payload_to_db(payload,
storage_type,
True,
cancellable)
except Exception as e:
Logger.warning("DeezerSearch::get(): %s", e)
if not cancellable.is_cancelled():
emit_signal(self, "finished")
def get(self, search, storage_type, cancellable):
"""
Get albums for search
We need a thread because we are going to populate DB
@param search as str
@param storage_type as StorageType
@param cancellable as Gio.Cancellable
"""
if not get_network_available("MUSICBRAINZ"):
emit_signal(self, "finished")
return
try:
uri = "http://musicbrainz.org/ws/2/release/?fmt=json&query=%s" %\
search
(status, data) = App().task_helper.load_uri_content_sync(
uri, cancellable)
if status:
decode = json.loads(data.decode("utf-8"))
for release in decode["releases"]:
payload = self.lollypop_album_payload(release)
self.save_album_payload_to_db(payload,
storage_type,
True,
cancellable)
except Exception as e:
Logger.warning("MusicBrainzSearch::get(): %s", e)
if not cancellable.is_cancelled():
emit_signal(self, "finished")
def search_similar_albums(self, cancellable):
"""
Add similar albums to DB
@param cancellable as Gio.Cancellable
"""
Logger.info("Get similar albums from Spotify")
from lollypop.similars_spotify import SpotifySimilars
similars = SpotifySimilars()
try:
storage_type = get_default_storage_type()
artists = App().artists.get_randoms(
self.MAX_ITEMS_PER_STORAGE_TYPE, storage_type)
artist_names = [name for (aid, name, sortname) in artists]
similar_ids = similars.get_similar_artist_ids(
artist_names, cancellable)
# Add albums
shuffle(similar_ids)
for similar_id in similar_ids[:self.MAX_ITEMS_PER_STORAGE_TYPE]:
albums_payload = self.__get_artist_albums_payload(
similar_id, cancellable)
shuffle(albums_payload)
for album in albums_payload:
if cancellable.is_cancelled():
raise Exception("Cancelled")
lollypop_payload = SpotifyWebHelper.lollypop_album_payload(
self, album)
self.save_album_payload_to_db(lollypop_payload,
StorageType.SPOTIFY_SIMILARS,
True, cancellable)
break
except Exception as e:
Logger.warning("SpotifyWebService::search_similar_albums(): %s", e)
def _get_deezer_artist_artwork_uri(self, artist, cancellable=None):
"""
Get artist artwork using Deezer
@param artist as str
@param cancellable as Gio.Cancellable
@return uri as str
@tread safe
"""
if not get_network_available("DEEZER"):
return []
try:
artist_formated = GLib.uri_escape_string(
artist, None, True).replace(" ", "+")
uri = "https://api.deezer.com/search/artist/?" +\
"q=%s&output=json&index=0&limit=1" % artist_formated
(status, data) = App().task_helper.load_uri_content_sync(
uri, cancellable)
if status:
uri = None
decode = json.loads(data.decode("utf-8"))
uri = decode["data"][0]["picture_xl"]
return [uri]
except Exception as e:
Logger.warning("%s %s", e, artist)
Logger.warning(
"DownloaderArt::_get_deezer_artist_artwork_uri(): %s", data)
return []
def _on_eventbox_realize(self, eventbox):
"""
Show hand cursor over
"""
try:
eventbox.get_window().set_cursor(Gdk.Cursor(Gdk.CursorType.HAND2))
except:
Logger.warning(_("You are using a broken cursor theme!"))
def decode_lyrics(bytes):
try:
frame = FrameLangTag(bytes)
if frame.key == "USLT":
return frame.string
except Exception as e:
Logger.warning("TagReader::get_lyrics(): %s", e)
return None
def _on_label_realize(self, eventbox):
"""
@param eventbox as Gtk.EventBox
"""
try:
eventbox.get_window().set_cursor(Gdk.Cursor(Gdk.CursorType.HAND2))
except:
Logger.warning(_("You are using a broken cursor theme!"))
def decodeUnicode(bites, encoding):
codec = id3EncodingToString(encoding)
Logger.debug("Unicode encoding: %s" % codec)
if (codec.startswith("utf_16") and len(bites) % 2 != 0
and bites[-1:] == b"\x00"):
# Catch and fix bad utf16 data, it is everywhere.
Logger.warning("Fixing utf16 data with extra zero bytes")
bites = bites[:-1]
return bites.decode(codec).rstrip("\x00")
def select_first(self):
"""
Select first available item
"""
try:
self._listbox.unselect_all()
row = self._listbox.get_children()[0]
row.activate()
except Exception as e:
Logger.warning("SelectionList::select_first(): %s", e)
def __on_realize(self, widget):
"""
Change cursor over eventbox
@param widget as Gtk.Widget
"""
try:
window = widget.get_window()
if window is not None:
window.set_cursor(Gdk.Cursor(Gdk.CursorType.HAND2))
except:
Logger.warning(_("You are using a broken cursor theme!"))
def on_realize(widget):
"""
Set cursor on widget
@param widget as Gtk.Widget
"""
try:
window = widget.get_window()
if window is not None:
window.set_cursor(Gdk.Cursor(Gdk.CursorType.HAND2))
except:
Logger.warning(_("You are using a broken cursor theme!"))
def decode_lyrics(bytes_list, encoding):
lyrics = []
try:
for frame in bytes_list:
(l, t) = splitUnicode(frame, encoding)
if l:
lyrics.append((decodeUnicode(l, encoding),
int.from_bytes(t[1:4], "big")))
except Exception as e:
Logger.warning("TagReader::get_synced_lyrics1(): %s", e)
return lyrics
def _on_label_realize(self, eventbox):
"""
Change cursor on label
@param eventbox as Gtk.EventBox
"""
try:
if len(self._artist_ids) == 1:
eventbox.get_window().set_cursor(
Gdk.Cursor(Gdk.CursorType.HAND2))
except:
Logger.warning(_("You are using a broken cursor theme!"))
def __on_eventbox_realize(self, eventbox):
"""
Change cursor over eventbox
@param eventbox as Gdk.Eventbox
"""
try:
window = eventbox.get_window()
if window is not None:
window.set_cursor(Gdk.Cursor(Gdk.CursorType.HAND2))
except:
Logger.warning(_("You are using a broken cursor theme!"))
