def __init__(self):
GObject.GObject.__init__(self)
ScannerTagReader.__init__(self)
self._inotify = None
if Lp.settings.get_value('auto-update'):
self._inotify = Inotify()
self._is_empty = True
self._in_thread = False
self._is_locked = False
self._progress = None
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
ScannerTagReader.__init__(self)
self._thread = None
self._inotify = None
if Lp().settings.get_value('auto-update'):
self._inotify = Inotify()
self._progress = None
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
TagReader.__init__(self)
self.__thread = None
self.__history = None
if Lp().settings.get_value('auto-update'):
self.__inotify = Inotify()
else:
self.__inotify = None
Lp().albums.update_max_count()
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
ScannerTagReader.__init__(self)
self._albums_popularity = {}
self._albums_mtime = {}
self._tracks_popularity = {}
self._tracks_ltime = {}
self._thread = None
self._inotify = None
if Lp.settings.get_value("auto-update"):
self._inotify = Inotify()
self._is_empty = True
self._progress = None
class CollectionScanner(GObject.GObject, TagReader):
"""
Scan user music collection
"""
__gsignals__ = {
'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
'artist-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
'genre-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
'album-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool))
}
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
TagReader.__init__(self)
self.__thread = None
self.__history = None
if Lp().settings.get_value('auto-update'):
self.__inotify = Inotify()
else:
self.__inotify = None
Lp().albums.update_max_count()
def update(self):
"""
Update database
"""
if not self.is_locked():
uris = Lp().settings.get_music_uris()
if not uris:
return
Lp().window.progress.add(self)
Lp().window.progress.set_fraction(0.0, self)
self.__thread = Thread(target=self.__scan, args=(uris, ))
self.__thread.daemon = True
self.__thread.start()
def clean_charts(self):
"""
Clean charts in db
"""
self.__thread = Thread(target=self.__clean_charts)
self.__thread.daemon = True
self.__thread.start()
def is_locked(self):
"""
Return True if db locked
"""
return self.__thread is not None and self.__thread.isAlive()
def stop(self):
"""
Stop scan
"""
self.__thread = None
#######################
# PRIVATE #
#######################
def __clean_charts(self):
"""
Clean charts in db
"""
track_ids = Lp().tracks.get_old_charts_track_ids(int(time()))
Lp().db.del_tracks(track_ids)
self.stop()
def __get_objects_for_uris(self, uris):
"""
Return all tracks/dirs for uris
@param uris as string
@return (track uri as [str], track dirs as [str],
ignore dirs as [str])
"""
tracks = []
ignore_dirs = []
track_dirs = list(uris)
walk_uris = list(uris)
while walk_uris:
uri = walk_uris.pop(0)
empty = True
try:
d = Lio.File.new_for_uri(uri)
infos = d.enumerate_children(
'standard::name,standard::type,standard::is-hidden',
Gio.FileQueryInfoFlags.NONE, None)
except Exception as e:
print("CollectionScanner::__get_objects_for_uris():", e)
continue
for info in infos:
f = infos.get_child(info)
child_uri = f.get_uri()
empty = False
if info.get_is_hidden():
continue
elif info.get_file_type() == Gio.FileType.DIRECTORY:
track_dirs.append(child_uri)
walk_uris.append(child_uri)
else:
try:
f = Lio.File.new_for_uri(child_uri)
if is_pls(f):
pass
elif is_audio(f):
tracks.append(child_uri)
else:
debug("%s not detected as a music file" %
child_uri)
except Exception as e:
print(
"CollectionScanner::"
"__get_objects_for_uris():", e)
# If a root uri is empty
# Ensure user is not doing something bad
if empty and uri in uris:
ignore_dirs.append(uri)
return (tracks, track_dirs, ignore_dirs)
def __update_progress(self, current, total):
"""
Update progress bar status
@param scanned items as int, total items as int
"""
Lp().window.progress.set_fraction(current / total, self)
def __finish(self):
"""
Notify from main thread when scan finished
"""
Lp().window.progress.set_fraction(1.0, self)
self.stop()
self.emit("scan-finished")
# Update max count value
Lp().albums.update_max_count()
if Lp().settings.get_value('artist-artwork'):
Lp().art.cache_artists_info()
def __scan(self, uris):
"""
Scan music collection for music files
@param uris as [string], uris to scan
@thread safe
"""
gst_message = None
if self.__history is None:
self.__history = History()
mtimes = Lp().tracks.get_mtimes()
(new_tracks, new_dirs, ignore_dirs) = self.__get_objects_for_uris(uris)
orig_tracks = Lp().tracks.get_uris(ignore_dirs)
was_empty = len(orig_tracks) == 0
if ignore_dirs:
if Lp().notify is not None:
Lp().notify.send(_("Lollypop is detecting an empty folder."),
_("Check your music settings."))
count = len(new_tracks) + len(orig_tracks)
# Add monitors on dirs
if self.__inotify is not None:
for d in new_dirs:
if d.startswith("file://"):
self.__inotify.add_monitor(d)
with SqlCursor(Lp().db) as sql:
i = 0
# Look for new files/modified files
try:
to_add = []
for uri in new_tracks:
if self.__thread is None:
return
try:
GLib.idle_add(self.__update_progress, i, count)
f = Lio.File.new_for_uri(uri)
info = f.query_info("time::modified",
Gio.FileQueryInfoFlags.NONE, None)
mtime = int(
info.get_attribute_as_string("time::modified"))
# If songs exists and mtime unchanged, continue,
# else rescan
if uri in orig_tracks:
orig_tracks.remove(uri)
i += 1
if mtime <= mtimes[uri]:
i += 1
continue
else:
self.__del_from_db(uri)
# On first scan, use modification time
# Else, use current time
if not was_empty:
mtime = int(time())
to_add.append((uri, mtime))
except Exception as e:
print("CollectionScanner::__scan(mtime):", e)
# Clean deleted files
# Now because we need to populate history
for uri in orig_tracks:
i += 1
GLib.idle_add(self.__update_progress, i, count)
self.__del_from_db(uri)
# Add files to db
for (uri, mtime) in to_add:
try:
debug("Adding file: %s" % uri)
i += 1
GLib.idle_add(self.__update_progress, i, count)
self.__add2db(uri, mtime)
except Exception as e:
print("CollectionScanner::__scan(add):", e, uri)
if e.message != gst_message:
gst_message = e.message
if Lp().notify is not None:
Lp().notify.send(gst_message, uri)
sql.commit()
except Exception as e:
print("CollectionScanner::__scan():", e)
GLib.idle_add(self.__finish)
del self.__history
self.__history = None
def __add2db(self, uri, mtime):
"""
Add new file to db with informations
@param uri as string
@param mtime as int
@return track id as int
"""
f = Lio.File.new_for_uri(uri)
debug("CollectionScanner::add2db(): Read tags")
info = self.get_info(uri)
tags = info.get_tags()
name = f.get_basename()
title = self.get_title(tags, name)
artists = self.get_artists(tags)
composers = self.get_composers(tags)
performers = self.get_performers(tags)
a_sortnames = self.get_artist_sortnames(tags)
aa_sortnames = self.get_album_artist_sortnames(tags)
album_artists = self.get_album_artist(tags)
album_name = self.get_album_name(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
discname = self.get_discname(tags)
tracknumber = self.get_tracknumber(tags, name)
year = self.get_original_year(tags)
if year is None:
year = self.get_year(tags)
duration = int(info.get_duration() / 1000000000)
# If no artists tag, use album artist
if artists == '':
artists = album_artists
# if artists is always null, no album artists too,
# use composer/performer
if artists == '':
artists = performers
album_artists = composers
if artists == '':
artists = album_artists
if artists == '':
artists = _("Unknown")
debug("CollectionScanner::add2db(): Restore stats")
# Restore stats
(track_pop, track_rate, track_ltime, album_mtime, loved, album_pop,
album_rate) = self.__history.get(name, duration)
# If nothing in stats, use track mtime
if album_mtime == 0:
album_mtime = mtime
debug("CollectionScanner::add2db(): Add artists %s" % artists)
artist_ids = self.add_artists(artists, album_artists, a_sortnames)
debug("CollectionScanner::add2db(): "
"Add album artists %s" % album_artists)
album_artist_ids = self.add_album_artists(album_artists, aa_sortnames)
new_artist_ids = list(set(album_artist_ids) | set(artist_ids))
debug("CollectionScanner::add2db(): Add album: "
"%s, %s" % (album_name, album_artist_ids))
(album_id, new_album) = self.add_album(album_name, album_artist_ids,
uri, loved, album_pop,
album_rate, False)
genre_ids = self.add_genres(genres)
# Add track to db
debug("CollectionScanner::add2db(): Add track")
track_id = Lp().tracks.add(title, uri, duration, tracknumber,
discnumber, discname, album_id, year,
track_pop, track_rate, track_ltime)
debug("CollectionScanner::add2db(): Update tracks")
self.update_track(track_id, artist_ids, genre_ids, mtime)
self.update_album(album_id, album_artist_ids, genre_ids, album_mtime,
year)
if new_album:
with SqlCursor(Lp().db) as sql:
sql.commit()
for genre_id in genre_ids:
GLib.idle_add(self.emit, 'genre-updated', genre_id, True)
for artist_id in new_artist_ids:
GLib.idle_add(self.emit, 'artist-updated', artist_id, True)
return track_id
def __del_from_db(self, uri):
"""
Delete track from db
@param uri as str
"""
try:
f = Lio.File.new_for_uri(uri)
name = f.get_basename()
track_id = Lp().tracks.get_id_by_uri(uri)
album_id = Lp().tracks.get_album_id(track_id)
genre_ids = Lp().tracks.get_genre_ids(track_id)
album_artist_ids = Lp().albums.get_artist_ids(album_id)
artist_ids = Lp().tracks.get_artist_ids(track_id)
popularity = Lp().tracks.get_popularity(track_id)
rate = Lp().tracks.get_rate(track_id)
ltime = Lp().tracks.get_ltime(track_id)
mtime = Lp().albums.get_mtime(album_id)
duration = Lp().tracks.get_duration(track_id)
album_popularity = Lp().albums.get_popularity(album_id)
album_rate = Lp().albums.get_rate(album_id)
loved = Lp().albums.get_loved(album_id)
uri = Lp().tracks.get_uri(track_id)
self.__history.add(name, duration, popularity, rate, ltime, mtime,
loved, album_popularity, album_rate)
Lp().tracks.remove(track_id)
Lp().tracks.clean(track_id)
deleted = Lp().albums.clean(album_id)
if deleted:
with SqlCursor(Lp().db) as sql:
sql.commit()
GLib.idle_add(self.emit, 'album-updated', album_id, True)
for artist_id in album_artist_ids + artist_ids:
Lp().artists.clean(artist_id)
GLib.idle_add(self.emit, 'artist-updated', artist_id, False)
for genre_id in genre_ids:
Lp().genres.clean(genre_id)
GLib.idle_add(self.emit, 'genre-updated', genre_id, False)
except Exception as e:
print("CollectionScanner::__del_from_db:", e)
class CollectionScanner(GObject.GObject, TagReader):
"""
Scan user music collection
"""
__gsignals__ = {
"scan-finished": (GObject.SignalFlags.RUN_FIRST, None, (bool, )),
"artist-updated": (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
"genre-updated": (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
"album-updated": (GObject.SignalFlags.RUN_FIRST, None, (int, bool))
}
_WEB_COLLECTION = GLib.get_user_data_dir() + "/lollypop/web_collection"
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
TagReader.__init__(self)
self.__thread = None
self.__history = History()
self.__disable_compilations = True
if App().settings.get_value("auto-update"):
self.__inotify = Inotify()
else:
self.__inotify = None
App().albums.update_max_count()
create_dir(self._WEB_COLLECTION)
def update(self, scan_type, uris=[]):
"""
Update database
@param scan_type as ScanType
@param uris as [str]
"""
App().lookup_action("update_db").set_enabled(False)
# Stop previous scan
if self.is_locked() and scan_type != ScanType.EPHEMERAL:
self.stop()
GLib.timeout_add(250, self.update, scan_type, uris)
else:
self.__disable_compilations = not App().settings.get_value(
"show-compilations")
if scan_type == ScanType.FULL:
uris = App().settings.get_music_uris()
if not uris:
return
# Register to progressbar
if scan_type != ScanType.EPHEMERAL:
App().window.container.progress.add(self)
App().window.container.progress.set_fraction(0, self)
# Launch scan in a separate thread
self.__thread = Thread(target=self.__scan, args=(scan_type, uris))
self.__thread.daemon = True
self.__thread.start()
def update_album(self, album_id, album_artist_ids, genre_ids, year,
timestamp):
"""
Update album artists based on album-artist and artist tags
This code auto handle compilations: empty "album artist" with
different artists
@param album_id as int
@param album_artist_ids as [int]
@param genre_ids as [int]
@param year as int
@param timestamp as int
@commit needed
"""
if album_artist_ids:
# Update UI based on previous artist calculation
mtime = App().albums.get_mtime(album_id)
if mtime != 0:
for artist_id in album_artist_ids:
GLib.idle_add(self.emit, "artist-updated", artist_id, True)
App().albums.set_artist_ids(album_id, album_artist_ids)
# Set artist ids based on content
else:
new_album_artist_ids = App().albums.calculate_artist_ids(album_id)
App().albums.set_artist_ids(album_id, new_album_artist_ids)
# Update album genres
for genre_id in genre_ids:
App().albums.add_genre(album_id, genre_id)
# Update year based on tracks
year = App().tracks.get_year_for_album(album_id)
App().albums.set_year(album_id, year)
timestamp = App().tracks.get_timestamp_for_album(album_id)
App().albums.set_timestamp(album_id, timestamp)
def save_track(self, genres, artists, a_sortnames, mb_artist_id,
album_artists, aa_sortnames, mb_album_artist_id, album_name,
mb_album_id, uri, album_loved, album_pop, album_rate,
album_synced, album_mtime, title, duration, tracknumber,
discnumber, discname, year, timestamp, track_mtime,
track_pop, track_rate, track_loved, track_ltime,
mb_track_id, bpm):
"""
Add track to DB
@param genres as str/None
@param artists as str
@param a_sortnames as str
@param mb_artist_id as str
@param album_artists as str
@param aa_sortnames as str
@param mb_album_artist_id as str
@param album_name as str
@param mb_album_id as str
@param uri as str
@param album_loved as int
@param album_pop as int
@param album_rate as int
@param album_synced as int
@param album_mtime as int
@param title as str
@param duration as int
@param tracknumber as int
@param discnumber as int
@param discname as str
@param year as int
@param timestamp as int
@param track_mtime as int
@param track_pop as int
@param track_rate as int
@param track_loved as int
@param track_ltime as int
@param mb_track_id as str
@param bpm as int
"""
Logger.debug("CollectionScanner::save_track(): Add artists %s" %
artists)
artist_ids = self.add_artists(artists, a_sortnames, mb_artist_id)
Logger.debug("CollectionScanner::save_track(): "
"Add album artists %s" % album_artists)
album_artist_ids = self.add_artists(album_artists, aa_sortnames,
mb_album_artist_id)
# User does not want compilations
if self.__disable_compilations and not album_artist_ids:
album_artist_ids = artist_ids
missing_artist_ids = list(set(album_artist_ids) - set(artist_ids))
# https://github.com/gnumdk/lollypop/issues/507#issuecomment-200526942
# Special case for broken tags
# Can't do more because don't want to break split album behaviour
if len(missing_artist_ids) == len(album_artist_ids):
artist_ids += missing_artist_ids
Logger.debug("CollectionScanner::save_track(): Add album: "
"%s, %s" % (album_name, album_artist_ids))
(album_added,
album_id) = self.add_album(album_name, mb_album_id, album_artist_ids,
uri, album_loved, album_pop, album_rate,
album_synced, album_mtime)
if genres is None:
genre_ids = [Type.WEB]
else:
genre_ids = self.add_genres(genres)
# Add track to db
Logger.debug("CollectionScanner::save_track(): Add track")
track_id = App().tracks.add(title, uri, duration, tracknumber,
discnumber, discname, album_id, year,
timestamp, track_pop, track_rate,
track_loved, track_ltime, track_mtime,
mb_track_id, bpm)
Logger.debug("CollectionScanner::save_track(): Update track")
self.update_track(track_id, artist_ids, genre_ids)
Logger.debug("CollectionScanner::save_track(): Update album")
SqlCursor.commit(App().db)
self.update_album(album_id, album_artist_ids, genre_ids, year,
timestamp)
SqlCursor.commit(App().db)
for genre_id in genre_ids:
# Be sure to not send Type.WEB
if genre_id >= 0:
GLib.idle_add(self.emit, "genre-updated", genre_id, True)
if album_added:
GLib.idle_add(self.emit, "album-updated", album_id, True)
return (track_id, album_id)
def update_track(self, track_id, artist_ids, genre_ids):
"""
Set track artists/genres
@param track_id as int
@param artist_ids as [int]
@param genre_ids as [int]
@commit needed
"""
# Set artists/genres for track
for artist_id in artist_ids:
App().tracks.add_artist(track_id, artist_id)
for genre_id in genre_ids:
App().tracks.add_genre(track_id, genre_id)
def del_from_db(self, uri, backup, notify=True):
"""
Delete track from db
@param uri as str
@param backup as bool
@param notify as bool => send signal about cleanup
@return (popularity, ltime, mtime,
loved album, album_popularity)
"""
try:
track_id = App().tracks.get_id_by_uri(uri)
duration = App().tracks.get_duration(track_id)
album_id = App().tracks.get_album_id(track_id)
genre_ids = App().tracks.get_genre_ids(track_id)
album_artist_ids = App().albums.get_artist_ids(album_id)
artist_ids = App().tracks.get_artist_ids(track_id)
track_pop = App().tracks.get_popularity(track_id)
track_rate = App().tracks.get_rate(track_id)
track_ltime = App().tracks.get_ltime(track_id)
album_mtime = App().tracks.get_mtime(track_id)
track_loved = App().tracks.get_loved(track_id)
album_pop = App().albums.get_popularity(album_id)
album_rate = App().albums.get_rate(album_id)
album_loved = App().albums.get_loved(album_id)
album_synced = App().albums.get_synced(album_id)
# Force genre for album
App().albums.set_genre_ids(album_id, genre_ids)
if backup:
f = Gio.File.new_for_uri(uri)
name = f.get_basename()
self.__history.add(name, duration, track_pop, track_rate,
track_ltime, album_mtime, track_loved,
album_loved, album_pop, album_rate,
album_synced)
App().tracks.remove(track_id)
App().albums.clean()
App().genres.clean()
App().artists.clean()
if notify:
if App().albums.get_name(album_id) is None:
GLib.idle_add(self.emit, "album-updated", album_id, False)
for artist_id in album_artist_ids + artist_ids:
GLib.idle_add(self.emit, "artist-updated", artist_id,
False)
for genre_id in genre_ids:
GLib.idle_add(self.emit, "genre-updated", genre_id, False)
return (track_pop, track_rate, track_ltime, album_mtime,
track_loved, album_loved, album_pop, album_rate)
except Exception as e:
Logger.error("CollectionScanner::del_from_db: %s" % e)
def is_locked(self):
"""
Return True if db locked
"""
return self.__thread is not None and self.__thread.isAlive()
def stop(self):
"""
Stop scan
"""
self.__thread = None
@property
def inotify(self):
"""
Get Inotify object
@return Inotify
"""
return self.__inotify
#######################
# PRIVATE #
#######################
def __update_progress(self, current, total):
"""
Update progress bar status
@param scanned items as int, total items as int
"""
GLib.idle_add(App().window.container.progress.set_fraction,
current / total, self)
def __finish(self, modifications):
"""
Notify from main thread when scan finished
@param modifications as bool
"""
App().lookup_action("update_db").set_enabled(True)
App().window.container.progress.set_fraction(1.0, self)
self.stop()
self.emit("scan-finished", modifications)
# Update max count value
App().albums.update_max_count()
def __add_monitor(self, dirs):
"""
Monitor any change in a list of directory
@param dirs as str or list of directory to be monitored
"""
if self.__inotify is None:
return
# Add monitors on dirs
for d in dirs:
# Handle a stop request
if self.__thread is None:
break
if d.startswith("file://"):
self.__inotify.add_monitor(d)
def __import_web_tracks(self):
"""
Import locally saved web tracks
"""
try:
# Directly add files, walk through directories
f = Gio.File.new_for_path(self._WEB_COLLECTION)
infos = f.enumerate_children(SCAN_QUERY_INFO,
Gio.FileQueryInfoFlags.NONE, None)
for info in infos:
f = infos.get_child(info)
if info.get_is_hidden():
continue
elif info.get_file_type() == Gio.FileType.DIRECTORY:
pass
else:
(status, content, tag) = f.load_contents()
data = json.loads(content)
self.save_track(
None, ";".join(data["artists"]), "", "", ";".join(
data["album_artists"]), "", "", data["album_name"],
"", data["uri"], data["album_loved"],
data["album_popularity"], data["album_rate"], 0, -1,
data["title"], data["duration"], data["tracknumber"],
data["discnumber"], data["discname"], data["year"],
data["timestamp"], -1, data["track_popularity"],
data["track_rate"], data["track_loved"], 0, "", 0)
infos.close(None)
except Exception as e:
Logger.error("CollectionScanner::__import_web_tracks(): %s", e)
@profile
def __get_objects_for_uris(self, scan_type, uris):
"""
Get all tracks and dirs in uris
@param scan_type as ScanType
@param uris as string
@return (tracks [mtimes: int, uri: str], dirs as [uri: str])
"""
files = []
dirs = []
walk_uris = []
# Check collection exists
for uri in uris:
f = Gio.File.new_for_uri(uri)
if f.query_exists():
walk_uris.append(uri)
else:
return (None, None)
while walk_uris:
uri = walk_uris.pop(0)
try:
# Directly add files, walk through directories
f = Gio.File.new_for_uri(uri)
info = f.query_info(SCAN_QUERY_INFO,
Gio.FileQueryInfoFlags.NONE, None)
if info.get_file_type() == Gio.FileType.DIRECTORY:
dirs.append(uri)
infos = f.enumerate_children(SCAN_QUERY_INFO,
Gio.FileQueryInfoFlags.NONE,
None)
for info in infos:
f = infos.get_child(info)
child_uri = f.get_uri()
if info.get_is_hidden():
continue
elif info.get_file_type() == Gio.FileType.DIRECTORY:
dirs.append(child_uri)
walk_uris.append(child_uri)
else:
mtime = get_mtime(info)
files.append((mtime, child_uri))
infos.close(None)
# Only happens if files passed as args
else:
mtime = get_mtime(info)
files.append((mtime, uri))
except Exception as e:
Logger.error(
"CollectionScanner::__get_objects_for_uris(): %s" % e)
files.sort(reverse=True)
return (files, dirs)
@profile
def __scan(self, scan_type, uris):
"""
Scan music collection for music files
@param scan_type as ScanType
@param uris as [str]
@thread safe
"""
if not App().tracks.get_mtimes():
self.__import_web_tracks()
(files, dirs) = self.__get_objects_for_uris(scan_type, uris)
if files is None:
if App().notify is not None:
App().notify.send(_("Scan disabled, missing collection"))
return
if scan_type == ScanType.NEW_FILES:
db_uris = App().tracks.get_uris(uris)
else:
db_uris = App().tracks.get_uris()
new_tracks = self.__scan_files(files, db_uris, scan_type)
if scan_type != ScanType.EPHEMERAL:
self.__add_monitor(dirs)
GLib.idle_add(self.__finish, new_tracks)
if scan_type == ScanType.EPHEMERAL:
App().player.play_uris(new_tracks)
def __scan_to_handle(self, uri):
"""
Check if file has to be handle by scanner
@param f as Gio.File
@return bool
"""
try:
f = Gio.File.new_for_uri(uri)
# Scan file
if App().settings.get_value("import-playlists") and is_pls(f):
# Handle playlist
App().playlists.import_tracks(f)
elif is_audio(f):
return True
else:
Logger.debug("Not detected as a music file: %s" % f.get_uri())
except Exception as e:
Logger.error("CollectionScanner::__scan_to_handle(): %s" % e)
return False
@profile
def __scan_files(self, files, db_uris, scan_type):
"""
Scan music collection for new audio files
@param files as [str]
@param db_uris as [str]
@param scan_type as ScanType
@return new track uris as [str]
@thread safe
"""
SqlCursor.add(App().db)
i = 0
# New tracks present in collection
new_tracks = []
# Get mtime of all tracks to detect which has to be updated
db_mtimes = App().tracks.get_mtimes()
count = len(files) + 1
try:
# Scan new files
for (mtime, uri) in files:
# Handle a stop request
if self.__thread is None and scan_type != ScanType.EPHEMERAL:
raise Exception("Scan add cancelled")
try:
if not self.__scan_to_handle(uri):
continue
if mtime > db_mtimes.get(uri, 0):
# If not saved, use 0 as mtime, easy delete on quit
if scan_type == ScanType.EPHEMERAL:
mtime = 0
# Do not use mtime if not intial scan
elif db_mtimes:
mtime = int(time())
Logger.debug("Adding file: %s" % uri)
self.__add2db(uri, mtime)
SqlCursor.allow_thread_execution(App().db)
new_tracks.append(uri)
except Exception as e:
Logger.error("CollectionScanner:: __scan_add_files: % s" %
e)
i += 1
self.__update_progress(i, count)
if scan_type != ScanType.EPHEMERAL and self.__thread is not None:
# We need to check files are always in collections
if scan_type == ScanType.FULL:
collections = App().settings.get_music_uris()
else:
collections = None
for uri in db_uris:
# Handle a stop request
if self.__thread is None:
raise Exception("Scan del cancelled")
in_collection = True
if collections is not None:
in_collection = False
for collection in collections:
if collection in uri:
in_collection = True
break
f = Gio.File.new_for_uri(uri)
if not in_collection or not f.query_exists():
self.del_from_db(uri, True)
SqlCursor.allow_thread_execution(App().db)
except Exception as e:
Logger.warning("CollectionScanner:: __scan_files: % s" % e)
SqlCursor.commit(App().db)
SqlCursor.remove(App().db)
return new_tracks
def __add2db(self, uri, track_mtime):
"""
Add new file(or update one) to db with information
@param uri as string
@param track_mtime as int
@return track id as int
@warning, be sure SqlCursor is available for App().db
"""
f = Gio.File.new_for_uri(uri)
Logger.debug("CollectionScanner::add2db(): Read tags")
info = self.get_info(uri)
tags = info.get_tags()
name = f.get_basename()
title = self.get_title(tags, name)
version = self.get_version(tags)
artists = self.get_artists(tags)
composers = self.get_composers(tags)
performers = self.get_performers(tags)
remixers = self.get_remixers(tags)
if remixers != "":
artists += ";%s" % remixers
a_sortnames = self.get_artist_sortnames(tags)
aa_sortnames = self.get_album_artist_sortnames(tags)
album_artists = self.get_album_artists(tags)
album_name = self.get_album_name(tags)
album_synced = 0
mb_album_id = self.get_mb_album_id(tags)
mb_track_id = self.get_mb_track_id(tags)
mb_artist_id = self.get_mb_artist_id(tags)
mb_album_artist_id = self.get_mb_album_artist_id(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
discname = self.get_discname(tags)
tracknumber = self.get_tracknumber(tags, name)
track_popm = self.get_popm(tags)
bpm = self.get_bpm(tags)
(year, timestamp) = self.get_original_year(tags)
if year is None:
(year, timestamp) = self.get_year(tags)
duration = int(info.get_duration() / 1000000000)
if version != "":
title += " (%s)" % version
# If no artists tag, use album artist
if artists == "":
artists = album_artists
# if artists is always null, no album artists too,
# use composer/performer
if artists == "":
artists = performers
album_artists = composers
if artists == "":
artists = album_artists
if artists == "":
artists = _("Unknown")
Logger.debug("CollectionScanner::add2db(): Restore stats")
# Restore stats
track_id = App().tracks.get_id_by_uri(uri)
if track_id is None:
basename = f.get_basename()
track_id = App().tracks.get_id_by_basename_duration(
basename, duration)
if track_id is None:
(track_pop, track_rate, track_ltime, album_mtime, track_loved,
album_loved, album_pop, album_rate,
album_synced) = self.__history.get(name, duration)
# Delete track and restore from it
else:
(track_pop, track_rate, track_ltime, album_mtime, track_loved,
album_loved, album_pop,
album_rate) = self.del_from_db(uri, False)
# Prefer popm to internal rate
if track_popm != 0:
track_rate = track_popm
# If nothing in stats, use track mtime
if album_mtime == 0:
album_mtime = track_mtime
(track_id, album_id) = self.save_track(
genres, artists, a_sortnames, mb_artist_id, album_artists,
aa_sortnames, mb_album_artist_id, album_name, mb_album_id, uri,
album_loved, album_pop, album_rate, album_synced, album_mtime,
title, duration, tracknumber, discnumber, discname, year,
timestamp, track_mtime, track_pop, track_rate, track_loved,
track_ltime, mb_track_id, bpm)
return track_id
class CollectionScanner(GObject.GObject, ScannerTagReader):
"""
Scan user music collection
"""
__gsignals__ = {
'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
'artist-update': (GObject.SignalFlags.RUN_FIRST, None, (int, int)),
'genre-update': (GObject.SignalFlags.RUN_FIRST, None, (int, )),
'album-modified': (GObject.SignalFlags.RUN_FIRST, None, (int, ))
}
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
ScannerTagReader.__init__(self)
self._albums_popularity = {}
self._albums_mtime = {}
self._tracks_popularity = {}
self._tracks_ltime = {}
self._inotify = None
if Lp.settings.get_value('auto-update'):
self._inotify = Inotify()
self._is_empty = True
self._in_thread = False
self._is_locked = False
self._progress = None
def update(self, progress):
"""
Update database
@param progress as progress bar
"""
# Keep track of on file with missing codecs
self._missing_codecs = None
self.init_discover()
self._progress = progress
paths = Lp.settings.get_music_paths()
if not paths:
return
if not self._in_thread:
if Lp.notify is not None:
Lp.notify.send(_("Your music is updating"))
if self._progress is not None:
self._progress.show()
self._in_thread = True
self._is_locked = True
start_new_thread(self._scan, (paths, ))
def is_locked(self):
"""
Return True if db locked
"""
return self._is_locked
def stop(self):
"""
Stop scan
"""
if self._progress is not None:
self._progress.hide()
self._progress.set_fraction(0.0)
self._progress = None
self._in_thread = False
#######################
# PRIVATE #
#######################
def _get_objects_for_paths(self, paths):
"""
Return all tracks/dirs for paths
@param paths as string
@return ([tracks path], [dirs path], track count)
"""
tracks = []
track_dirs = list(paths)
count = 0
for path in paths:
for root, dirs, files in os.walk(path):
# Add dirs
for d in dirs:
track_dirs.append(os.path.join(root, d))
# Add files
for name in files:
filepath = os.path.join(root, name)
try:
f = Gio.File.new_for_path(filepath)
if is_pls(f):
pass
elif is_audio(f):
tracks.append(filepath)
count += 1
else:
debug("%s not detected as a music file" % filepath)
except Exception as e:
print("CollectionScanner::_get_objects_for_paths: %s" %
e)
return (tracks, track_dirs, count)
def _update_progress(self, current, total):
"""
Update progress bar status
@param scanned items as int, total items as int
"""
if self._progress is not None:
self._progress.set_fraction(current / total)
def _finish(self):
"""
Notify from main thread when scan finished
"""
if self._progress is not None:
self._progress.hide()
self._progress.set_fraction(0.0)
self._progress = None
self._in_thread = False
self._is_locked = False
self.emit("scan-finished")
if self._missing_codecs is not None:
Lp.player.load_external(GLib.filename_to_uri(self._missing_codecs))
Lp.player.play_first_external()
def _scan(self, paths):
"""
Scan music collection for music files
@param paths as [string], paths to scan
@thread safe
"""
sql = Lp.db.get_cursor()
mtimes = Lp.tracks.get_mtimes(sql)
orig_tracks = Lp.tracks.get_paths(sql)
self._is_empty = len(orig_tracks) == 0
if self._is_empty:
self._albums_popularity = Lp.db.get_albums_popularity()
self._albums_mtime = Lp.db.get_albums_mtime()
self._tracks_popularity = Lp.db.get_tracks_popularity()
self._tracks_ltime = Lp.db.get_tracks_ltime()
# Add monitors on dirs
(new_tracks, new_dirs, count) = self._get_objects_for_paths(paths)
if self._inotify is not None:
for d in new_dirs:
self._inotify.add_monitor(d)
i = 0
for filepath in new_tracks:
if not self._in_thread:
sql.close()
self._is_locked = False
return
GLib.idle_add(self._update_progress, i, count)
try:
mtime = int(os.path.getmtime(filepath))
if filepath not in orig_tracks:
try:
infos = self.get_infos(filepath)
debug("Adding file: %s" % filepath)
self._add2db(filepath, mtime, infos, sql)
except Exception as e:
debug("Error scanning: %s" % filepath)
string = "%s" % e
if string.startswith('gst-core-error-quark'):
self._missing_codecs = filepath
else:
# Update tags by removing song and readd it
if mtime != mtimes[filepath]:
self._del_from_db(filepath, sql)
infos = self.get_infos(filepath)
if infos is not None:
debug("Adding file: %s" % filepath)
self._add2db(filepath, mtime, infos, sql)
else:
print("Can't get infos for ", filepath)
orig_tracks.remove(filepath)
except Exception as e:
print(ascii(filepath))
print("CollectionScanner::_scan(): %s" % e)
i += 1
# Clean deleted files
if i > 0:
for filepath in orig_tracks:
self._del_from_db(filepath, sql)
sql.commit()
sql.close()
GLib.idle_add(self._finish)
def _add2db(self, filepath, mtime, infos, sql):
"""
Add new file to db with informations
@param filepath as string
@param file modification time as int
@param infos as GstPbutils.DiscovererInfo
@param sql as sqlite cursor
@return track id as int
"""
tags = infos.get_tags()
title = self.get_title(tags, filepath)
artists = self.get_artists(tags)
album_artist = self.get_album_artist(tags)
album_name = self.get_album_name(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
tracknumber = self.get_tracknumber(tags)
year = self.get_year(tags)
length = infos.get_duration() / 1000000000
(artist_ids,
new_artist_ids) = self.add_artists(artists, album_artist, sql)
(album_artist_id, new) = self.add_album_artist(album_artist, sql)
if new:
new_artist_ids.append(album_artist_id)
no_album_artist = False
if album_artist_id is None:
album_artist_id = artist_ids[0]
no_album_artist = True
# Search for datas to restore
search = "%s_%s" % (album_name, album_artist)
if search in self._albums_popularity:
popularity = self._albums_popularity[search]
del self._albums_popularity[search]
album_mtime = self._albums_mtime[search]
del self._albums_mtime[search]
else:
popularity = 0
album_mtime = mtime
album_id = self.add_album(album_name, album_artist_id, no_album_artist,
filepath, popularity, album_mtime, sql)
(genre_ids, new_genre_ids) = self.add_genres(genres, album_id, sql)
# Search for datas to restore
search = "%s_%s" % (title, album_artist)
if search in self._tracks_popularity:
popularity = self._tracks_popularity[search]
del self._tracks_popularity[search]
ltime = self._tracks_ltime[search]
del self._tracks_ltime[search]
else:
popularity = 0
ltime = 0
# Add track to db
Lp.tracks.add(title, filepath, length, tracknumber, discnumber,
album_id, year, popularity, ltime, mtime, sql)
self.update_year(album_id, sql)
track_id = Lp.tracks.get_id_by_path(filepath, sql)
self.update_track(track_id, artist_ids, genre_ids, sql)
# Notify about new artists/genres
if new_genre_ids or new_artist_ids:
sql.commit()
for genre_id in new_genre_ids:
GLib.idle_add(self.emit, 'genre-update', genre_id)
for artist_id in new_artist_ids:
GLib.idle_add(self.emit, 'artist-update', artist_id, album_id)
return track_id
def _del_from_db(self, filepath, sql):
"""
Delete track from db
@param filepath as string
@param sql as sqlite cursor
"""
track_id = Lp.tracks.get_id_by_path(filepath, sql)
album_id = Lp.tracks.get_album_id(track_id, sql)
genre_ids = Lp.tracks.get_genre_ids(track_id, sql)
album_artist_id = Lp.albums.get_artist_id(album_id, sql)
artist_ids = Lp.tracks.get_artist_ids(track_id, sql)
Lp.tracks.remove(filepath, sql)
Lp.tracks.clean(track_id, sql)
modified = Lp.albums.clean(album_id, sql)
if modified:
self.emit('album-modified', album_id)
for artist_id in [album_artist_id] + artist_ids:
Lp.artists.clean(artist_id, sql)
for genre_id in genre_ids:
Lp.genres.clean(genre_id, sql)
class CollectionScanner(GObject.GObject, ScannerTagReader):
"""
Scan user music collection
"""
__gsignals__ = {
'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
'artist-added': (GObject.SignalFlags.RUN_FIRST, None, (int, int)),
'genre-added': (GObject.SignalFlags.RUN_FIRST, None, (int,)),
'album-update': (GObject.SignalFlags.RUN_FIRST, None, (int,))
}
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
ScannerTagReader.__init__(self)
self._thread = None
self._inotify = None
self._history = None
if Lp().settings.get_value('auto-update'):
self._inotify = Inotify()
self._progress = None
def update(self, progress):
"""
Update database
@param progress as Gtk.Scale
"""
if not self.is_locked():
progress.show()
self._progress = progress
# Keep track of on file with missing codecs
self._missing_codecs = None
self.init_discover()
paths = Lp().settings.get_music_paths()
if not paths:
return
if Lp().notify is not None:
Lp().notify.send(_("Your music is updating"))
self._thread = Thread(target=self._scan, args=(paths,))
self._thread.daemon = True
self._thread.start()
def is_locked(self):
"""
Return True if db locked
"""
return self._thread is not None and self._thread.isAlive()
def stop(self):
"""
Stop scan
"""
self._thread = None
if self._progress is not None:
self._progress.hide()
self._progress.set_fraction(0.0)
self._progress = None
#######################
# PRIVATE #
#######################
def _get_objects_for_paths(self, paths):
"""
Return all tracks/dirs for paths
@param paths as string
@return ([tracks path], [dirs path], track count)
"""
tracks = []
track_dirs = list(paths)
count = 0
for path in paths:
for root, dirs, files in os.walk(path):
# Add dirs
for d in dirs:
track_dirs.append(os.path.join(root, d))
# Add files
for name in files:
filepath = os.path.join(root, name)
try:
f = Gio.File.new_for_path(filepath)
if is_pls(f):
pass
elif is_audio(f):
tracks.append(filepath)
count += 1
else:
debug("%s not detected as a music file" % filepath)
except Exception as e:
print("CollectionScanner::_get_objects_for_paths: %s"
% e)
return (tracks, track_dirs, count)
def _update_progress(self, current, total):
"""
Update progress bar status
@param scanned items as int, total items as int
"""
if self._progress is not None:
self._progress.set_fraction(current/total)
def _finish(self):
"""
Notify from main thread when scan finished
"""
self.stop()
self.emit("scan-finished")
if Lp().settings.get_value('artist-artwork'):
Lp().art.cache_artists_art()
if self._missing_codecs is not None:
Lp().player.load_external(
GLib.filename_to_uri(self._missing_codecs))
Lp().player.play_first_external()
def _scan(self, paths):
"""
Scan music collection for music files
@param paths as [string], paths to scan
@thread safe
"""
if self._history is None:
self._history = History()
mtimes = Lp().tracks.get_mtimes()
orig_tracks = Lp().tracks.get_paths()
was_empty = len(orig_tracks) == 0
(new_tracks, new_dirs, count) = self._get_objects_for_paths(paths)
count += len(orig_tracks)
# Add monitors on dirs
if self._inotify is not None:
for d in new_dirs:
self._inotify.add_monitor(d)
with SqlCursor(Lp().db) as sql:
i = 0
for filepath in new_tracks:
if self._thread is None:
return
GLib.idle_add(self._update_progress, i, count)
try:
# If songs exists and mtime unchanged, continue,
# else rescan
if filepath in orig_tracks:
orig_tracks.remove(filepath)
i += 1
mtime = int(os.path.getmtime(filepath))
if mtime <= mtimes[filepath]:
i += 1
continue
else:
self._del_from_db(filepath)
infos = self.get_infos(filepath)
# On first scan, use modification time
# Else, use current time
if was_empty:
mtime = int(os.path.getmtime(filepath))
else:
mtime = int(time())
debug("Adding file: %s" % filepath)
self._add2db(filepath, infos, mtime)
except Exception as e:
debug("Error scanning: %s, %s" % (filepath, e))
string = "%s" % e
if string.startswith('gst-core-error-quark'):
self._missing_codecs = filepath
i += 1
# Clean deleted files
for filepath in orig_tracks:
i += 1
GLib.idle_add(self._update_progress, i, count)
self._del_from_db(filepath)
sql.commit()
GLib.idle_add(self._finish)
del self._history
self._history = None
def _add2db(self, filepath, infos, mtime):
"""
Add new file to db with informations
@param filepath as string
@param infos as GstPbutils.DiscovererInfo
@param mtime as int
@return track id as int
"""
debug("CollectionScanner::add2db(): Read tags")
tags = infos.get_tags()
title = self.get_title(tags, filepath)
artists = self.get_artists(tags)
composers = self.get_composers(tags)
performers = self.get_performers(tags)
a_sortnames = self.get_artist_sortnames(tags)
aa_sortnames = self.get_album_artist_sortnames(tags)
album_artists = self.get_album_artist(tags)
album_name = self.get_album_name(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
discname = self.get_discname(tags)
tracknumber = self.get_tracknumber(tags)
year = self.get_year(tags)
duration = int(infos.get_duration()/1000000000)
name = GLib.path_get_basename(filepath)
# If no artists tag, use album artist
if artists == '':
artists = album_artists
# if artists is always null, no album artists too,
# use composer/performer
if artists == '':
artists = performers
album_artists = composers
if artists == '':
artists = album_artists
if artists == '':
artists = _("Unknown")
debug("CollectionScanner::add2db(): Restore stats")
# Restore stats
(track_pop, track_ltime, amtime, album_pop) = self._history.get(
name, duration)
# If nothing in stats, set mtime
if amtime == 0:
amtime = mtime
debug("CollectionScanner::add2db(): Add artists %s" % artists)
(artist_ids, new_artist_ids) = self.add_artists(artists,
album_artists,
a_sortnames)
debug("CollectionScanner::add2db(): "
"Add album artists %s" % album_artists)
(album_artist_ids, new_album_artist_ids) = self.add_album_artists(
album_artists,
aa_sortnames)
new_artist_ids += new_album_artist_ids
debug("CollectionScanner::add2db(): Add album: "
"%s, %s, %s" % (album_name, album_artist_ids, year))
(album_id, new_album) = self.add_album(album_name, album_artist_ids,
year, filepath,
album_pop, amtime)
(genre_ids, new_genre_ids) = self.add_genres(genres, album_id)
# Add track to db
debug("CollectionScanner::add2db(): Add track")
track_id = Lp().tracks.add(title, filepath, duration,
tracknumber, discnumber, discname,
album_id, year, track_pop,
track_ltime, mtime)
debug("CollectionScanner::add2db(): Update tracks")
self.update_track(track_id, artist_ids, genre_ids)
self.update_album(album_id, album_artist_ids, genre_ids)
# Notify about new artists/genres
if new_genre_ids or new_artist_ids:
with SqlCursor(Lp().db) as sql:
sql.commit()
for genre_id in new_genre_ids:
GLib.idle_add(self.emit, 'genre-added', genre_id)
for artist_id in new_artist_ids:
GLib.idle_add(self.emit, 'artist-added', artist_id, album_id)
return track_id
def _del_from_db(self, filepath):
"""
Delete track from db
@param filepath as str
"""
name = GLib.path_get_basename(filepath)
track_id = Lp().tracks.get_id_by_path(filepath)
album_id = Lp().tracks.get_album_id(track_id)
genre_ids = Lp().tracks.get_genre_ids(track_id)
album_artist_ids = Lp().albums.get_artist_ids(album_id)
artist_ids = Lp().tracks.get_artist_ids(track_id)
popularity = Lp().tracks.get_popularity(track_id)
ltime = Lp().tracks.get_ltime(track_id)
mtime = Lp().albums.get_mtime(album_id)
duration = Lp().tracks.get_duration(track_id)
album_popularity = Lp().albums.get_popularity(album_id)
self._history.add(name, duration, popularity,
ltime, mtime, album_popularity)
Lp().tracks.remove(track_id)
Lp().tracks.clean(track_id)
modified = Lp().albums.clean(album_id)
if modified:
with SqlCursor(Lp().db) as sql:
sql.commit()
GLib.idle_add(self.emit, 'album-update', album_id)
for artist_id in album_artist_ids + artist_ids:
Lp().artists.clean(artist_id)
for genre_id in genre_ids:
Lp().genres.clean(genre_id)
class CollectionScanner(GObject.GObject, ScannerTagReader):
"""
Scan user music collection
"""
__gsignals__ = {
'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
'artist-added': (GObject.SignalFlags.RUN_FIRST, None, (int, int)),
'genre-added': (GObject.SignalFlags.RUN_FIRST, None, (int, )),
'album-update': (GObject.SignalFlags.RUN_FIRST, None, (int, ))
}
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
ScannerTagReader.__init__(self)
self._thread = None
self._inotify = None
self._history = None
if Lp().settings.get_value('auto-update'):
self._inotify = Inotify()
self._progress = None
def update(self, progress):
"""
Update database
@param progress as Gtk.Scale
"""
if not self.is_locked():
progress.show()
self._progress = progress
# Keep track of on file with missing codecs
self._missing_codecs = None
self.init_discover()
paths = Lp().settings.get_music_paths()
if not paths:
return
if Lp().notify is not None:
Lp().notify.send(_("Your music is updating"))
self._thread = Thread(target=self._scan, args=(paths, ))
self._thread.daemon = True
self._thread.start()
def is_locked(self):
"""
Return True if db locked
"""
return self._thread is not None and self._thread.isAlive()
def stop(self):
"""
Stop scan
"""
self._thread = None
if self._progress is not None:
self._progress.hide()
self._progress.set_fraction(0.0)
self._progress = None
#######################
# PRIVATE #
#######################
def _get_objects_for_paths(self, paths):
"""
Return all tracks/dirs for paths
@param paths as string
@return ([tracks path], [dirs path], track count)
"""
tracks = []
track_dirs = list(paths)
count = 0
for path in paths:
for root, dirs, files in os.walk(path):
# Add dirs
for d in dirs:
track_dirs.append(os.path.join(root, d))
# Add files
for name in files:
filepath = os.path.join(root, name)
try:
f = Gio.File.new_for_path(filepath)
if is_pls(f):
pass
elif is_audio(f):
tracks.append(filepath)
count += 1
else:
debug("%s not detected as a music file" % filepath)
except Exception as e:
print("CollectionScanner::_get_objects_for_paths: %s" %
e)
return (tracks, track_dirs, count)
def _update_progress(self, current, total):
"""
Update progress bar status
@param scanned items as int, total items as int
"""
if self._progress is not None:
self._progress.set_fraction(current / total)
def _finish(self):
"""
Notify from main thread when scan finished
"""
self.stop()
self.emit("scan-finished")
if self._missing_codecs is not None:
Lp().player.load_external(
GLib.filename_to_uri(self._missing_codecs))
Lp().player.play_first_external()
def _scan(self, paths):
"""
Scan music collection for music files
@param paths as [string], paths to scan
@thread safe
"""
if self._history is None:
self._history = History()
mtimes = Lp().tracks.get_mtimes()
orig_tracks = Lp().tracks.get_paths()
was_empty = len(orig_tracks) == 0
(new_tracks, new_dirs, count) = self._get_objects_for_paths(paths)
count += len(orig_tracks)
# Add monitors on dirs
if self._inotify is not None:
for d in new_dirs:
self._inotify.add_monitor(d)
with SqlCursor(Lp().db) as sql:
i = 0
for filepath in new_tracks:
if self._thread is None:
return
GLib.idle_add(self._update_progress, i, count)
try:
# If songs exists and mtime unchanged, continue,
# else rescan
if filepath in orig_tracks:
orig_tracks.remove(filepath)
i += 1
mtime = int(os.path.getmtime(filepath))
if mtime <= mtimes[filepath]:
i += 1
continue
else:
self._del_from_db(filepath)
infos = self.get_infos(filepath)
# On first scan, use modification time
# Else, use current time
if was_empty:
mtime = int(os.path.getmtime(filepath))
else:
mtime = int(time())
debug("Adding file: %s" % filepath)
self._add2db(filepath, infos, mtime)
except Exception as e:
debug("Error scanning: %s, %s" % (filepath, e))
string = "%s" % e
if string.startswith('gst-core-error-quark'):
self._missing_codecs = filepath
i += 1
# Clean deleted files
for filepath in orig_tracks:
i += 1
GLib.idle_add(self._update_progress, i, count)
self._del_from_db(filepath)
sql.commit()
GLib.idle_add(self._finish)
del self._history
self._history = None
def _add2db(self, filepath, infos, mtime):
"""
Add new file to db with informations
@param filepath as string
@param infos as GstPbutils.DiscovererInfo
@param mtime as int
@return track id as int
"""
debug("CollectionScanner::add2db(): Read tags")
tags = infos.get_tags()
title = self.get_title(tags, filepath)
artists = self.get_artists(tags)
composers = self.get_composers(tags)
performers = self.get_performers(tags)
sortnames = self.get_artist_sortnames(tags)
album_artists = self.get_album_artist(tags)
album_name = self.get_album_name(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
tracknumber = self.get_tracknumber(tags)
year = self.get_year(tags)
duration = int(infos.get_duration() / 1000000000)
name = GLib.path_get_basename(filepath)
# If no artists tag, use album artist
if artists == '':
artists = album_artists
# if artists is always null, no album artists too,
# use composer/performer
if artists == '':
artists = performers
album_artists = composers
if artists == '':
artists = album_artists
debug("CollectionScanner::add2db(): Restore stats")
# Restore stats
(track_pop, track_ltime, amtime,
album_pop) = self._history.get(name, duration)
# If nothing in stats, set mtime
if amtime == 0:
amtime = mtime
debug("CollectionScanner::add2db(): Add artists %s" % artists)
(artist_ids,
new_artist_ids) = self.add_artists(artists, album_artists, sortnames)
debug("CollectionScanner::add2db(): "
"Add album artists %s" % album_artists)
(album_artist_ids,
new_album_artist_ids) = self.add_album_artists(album_artists)
new_artist_ids += new_album_artist_ids
# Check for album artist, if none, use first available artist
no_album_artist = False
if not album_artist_ids:
album_artist_ids = artist_ids
no_album_artist = True
debug("CollectionScanner::add2db(): Add album: "
"%s, %s, %s" % (album_name, album_artist_ids, year))
(album_id, new_album) = self.add_album(album_name, album_artist_ids,
no_album_artist, year, filepath,
album_pop, amtime)
(genre_ids, new_genre_ids) = self.add_genres(genres, album_id)
# Add track to db
debug("CollectionScanner::add2db(): Add track")
track_id = Lp().tracks.add(title, filepath, duration, tracknumber,
discnumber, album_id, year, track_pop,
track_ltime, mtime)
debug("CollectionScanner::add2db(): Update tracks")
self.update_album(album_id, album_artist_ids, genre_ids)
self.update_track(track_id, artist_ids, genre_ids)
# Notify about new artists/genres
if new_genre_ids or new_artist_ids:
with SqlCursor(Lp().db) as sql:
sql.commit()
for genre_id in new_genre_ids:
GLib.idle_add(self.emit, 'genre-added', genre_id)
for artist_id in new_artist_ids:
GLib.idle_add(self.emit, 'artist-added', artist_id, album_id)
return track_id
def _del_from_db(self, filepath):
"""
Delete track from db
@param filepath as str
"""
name = GLib.path_get_basename(filepath)
track_id = Lp().tracks.get_id_by_path(filepath)
album_id = Lp().tracks.get_album_id(track_id)
genre_ids = Lp().tracks.get_genre_ids(track_id)
album_artist_ids = Lp().albums.get_artist_ids(album_id)
artist_ids = Lp().tracks.get_artist_ids(track_id)
popularity = Lp().tracks.get_popularity(track_id)
ltime = Lp().tracks.get_ltime(track_id)
mtime = Lp().albums.get_mtime(album_id)
duration = Lp().tracks.get_duration(track_id)
album_popularity = Lp().albums.get_popularity(album_id)
self._history.add(name, duration, popularity, ltime, mtime,
album_popularity)
Lp().tracks.remove(track_id)
Lp().tracks.clean(track_id)
modified = Lp().albums.clean(album_id)
if modified:
with SqlCursor(Lp().db) as sql:
sql.commit()
GLib.idle_add(self.emit, 'album-update', album_id)
for artist_id in album_artist_ids + artist_ids:
Lp().artists.clean(artist_id)
for genre_id in genre_ids:
Lp().genres.clean(genre_id)
class CollectionScanner(GObject.GObject, ScannerTagReader):
"""
Scan user music collection
"""
__gsignals__ = {
'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
'artist-update': (GObject.SignalFlags.RUN_FIRST, None, (int, int)),
'genre-update': (GObject.SignalFlags.RUN_FIRST, None, (int,)),
'album-modified': (GObject.SignalFlags.RUN_FIRST, None, (int,))
}
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
ScannerTagReader.__init__(self)
self._thread = None
self._inotify = None
if Lp().settings.get_value('auto-update'):
self._inotify = Inotify()
self._progress = None
def update(self, progress):
"""
Update database
@param progress as Gtk.Scale
"""
if not self.is_locked():
progress.show()
self._progress = progress
# Keep track of on file with missing codecs
self._missing_codecs = None
self.init_discover()
paths = Lp().settings.get_music_paths()
if not paths:
return
if Lp().notify is not None:
Lp().notify.send(_("Your music is updating"))
self._thread = Thread(target=self._scan, args=(paths,))
self._thread.daemon = True
self._thread.start()
def is_locked(self):
"""
Return True if db locked
"""
return self._thread is not None and self._thread.isAlive()
def stop(self):
"""
Stop scan
"""
self._thread = None
if self._progress is not None:
self._progress.hide()
self._progress.set_fraction(0.0)
self._progress = None
#######################
# PRIVATE #
#######################
def _get_objects_for_paths(self, paths):
"""
Return all tracks/dirs for paths
@param paths as string
@return ([tracks path], [dirs path], track count)
"""
tracks = []
track_dirs = list(paths)
count = 0
for path in paths:
for root, dirs, files in os.walk(path):
# Add dirs
for d in dirs:
track_dirs.append(os.path.join(root, d))
# Add files
for name in files:
filepath = os.path.join(root, name)
try:
f = Gio.File.new_for_path(filepath)
if is_pls(f):
pass
elif is_audio(f):
tracks.append(filepath)
count += 1
else:
debug("%s not detected as a music file" % filepath)
except Exception as e:
print("CollectionScanner::_get_objects_for_paths: %s"
% e)
return (tracks, track_dirs, count)
def _update_progress(self, current, total):
"""
Update progress bar status
@param scanned items as int, total items as int
"""
if self._progress is not None:
self._progress.set_fraction(current/total)
def _finish(self):
"""
Notify from main thread when scan finished
"""
Lp().settings.set_value('db-mtime', GLib.Variant('i', int(time())))
self.stop()
self.emit("scan-finished")
if self._missing_codecs is not None:
Lp().player.load_external(
GLib.filename_to_uri(self._missing_codecs))
Lp().player.play_first_external()
def _scan(self, paths):
"""
Scan music collection for music files
@param paths as [string], paths to scan
@thread safe
"""
self._new_albums = []
mtimes = Lp().tracks.get_mtimes()
orig_tracks = Lp().tracks.get_paths()
is_empty = len(orig_tracks) == 0
# Add monitors on dirs
(new_tracks, new_dirs, count) = self._get_objects_for_paths(paths)
if self._inotify is not None:
for d in new_dirs:
self._inotify.add_monitor(d)
with SqlCursor(Lp().db) as sql:
i = 0
for filepath in new_tracks:
if self._thread is None:
return
GLib.idle_add(self._update_progress, i, count)
try:
mtime = int(os.path.getmtime(filepath))
if filepath not in orig_tracks:
try:
debug("Adding file: %s" % filepath)
infos = self.get_infos(filepath)
self._add2db(filepath, mtime, infos)
except Exception as e:
debug("Error scanning: %s, %s" % (filepath, e))
string = "%s" % e
if string.startswith('gst-core-error-quark'):
self._missing_codecs = filepath
else:
# Update tags by removing song and readd it
if mtime != mtimes[filepath]:
debug("Adding file: %s" % filepath)
infos = self.get_infos(filepath)
if infos is not None:
self._add2db(filepath, mtime, infos)
else:
print("Can't get infos for ", filepath)
else:
orig_tracks.remove(filepath)
except Exception as e:
print(ascii(filepath))
print("CollectionScanner::_scan(): %s" % e)
i += 1
# Restore stats for new albums
if not is_empty:
for album_id in self._new_albums:
duration = Lp().albums.get_duration(album_id, None)
count = Lp().albums.get_count(album_id, None)
value = Lp().albums.get_stats(duration, count)
if value is not None:
Lp().albums.set_popularity(album_id, value[0])
Lp().albums.set_mtime(album_id, value[1])
# Clean deleted files
for filepath in orig_tracks:
track_id = Lp().tracks.get_id_by_path(filepath)
self._del_from_db(track_id)
sql.commit()
GLib.idle_add(self._finish)
def _add2db(self, filepath, mtime, infos):
"""
Add new file to db with informations
@param filepath as string
@param file modification time as int
@param infos as GstPbutils.DiscovererInfo
@return track id as int
"""
tags = infos.get_tags()
title = self.get_title(tags, filepath)
artists = self.get_artists(tags)
sortname = self.get_artist_sortname(tags)
album_artist = self.get_album_artist(tags)
album_name = self.get_album_name(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
tracknumber = self.get_tracknumber(tags)
year = self.get_year(tags)
duration = int(infos.get_duration()/1000000000)
(artist_ids, new_artist_ids) = self.add_artists(artists,
album_artist,
sortname)
(album_artist_id, new) = self.add_album_artist(album_artist)
if new:
new_artist_ids.append(album_artist_id)
# Check for album artist, if none, use first available artist
no_album_artist = False
if album_artist_id is None:
album_artist_id = artist_ids[0]
no_album_artist = True
(album_id, new) = self.add_album(album_name, album_artist_id,
no_album_artist, year, filepath, 0,
mtime)
if new:
self._new_albums.append(album_id)
(genre_ids, new_genre_ids) = self.add_genres(genres, album_id)
# Restore stats
value = Lp().tracks.get_stats(filepath, duration)
if value is None:
popularity = 0
ltime = 0
else:
popularity = value[0]
ltime = value[1]
# Add track to db
track_id = Lp().tracks.add(title, filepath, duration,
tracknumber, discnumber,
album_id, year, popularity, ltime, mtime)
self.update_track(track_id, artist_ids, genre_ids)
# Notify about new artists/genres
if new_genre_ids or new_artist_ids:
with SqlCursor(Lp().db) as sql:
sql.commit()
for genre_id in new_genre_ids:
GLib.idle_add(self.emit, 'genre-update', genre_id)
for artist_id in new_artist_ids:
GLib.idle_add(self.emit, 'artist-update', artist_id, album_id)
return track_id
def _del_from_db(self, track_id):
"""
Delete track from db
@param track_id as int
"""
album_id = Lp().tracks.get_album_id(track_id)
genre_ids = Lp().tracks.get_genre_ids(track_id)
album_artist_id = Lp().albums.get_artist_id(album_id)
artist_ids = Lp().tracks.get_artist_ids(track_id)
Lp().tracks.remove(track_id)
Lp().tracks.clean(track_id)
modified = Lp().albums.clean(album_id)
if modified:
GLib.idle_add(self.emit, 'album-modified', album_id)
for artist_id in [album_artist_id] + artist_ids:
Lp().artists.clean(artist_id)
for genre_id in genre_ids:
Lp().genres.clean(genre_id)
class CollectionScanner(GObject.GObject, TagReader):
"""
Scan user music collection
"""
__gsignals__ = {
'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
'artist-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
'genre-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
'album-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool))
}
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
TagReader.__init__(self)
self.__thread = None
self.__history = None
if Lp().settings.get_value('auto-update'):
self.__inotify = Inotify()
else:
self.__inotify = None
Lp().albums.update_max_count()
def update(self):
"""
Update database
"""
if not self.is_locked():
uris = Lp().settings.get_music_uris()
if not uris:
return
Lp().window.progress.add(self)
Lp().window.progress.set_fraction(0.0, self)
self.__thread = Thread(target=self.__scan, args=(uris,))
self.__thread.daemon = True
self.__thread.start()
def clean_charts(self):
"""
Clean charts in db
"""
self.__thread = Thread(target=self.__clean_charts)
self.__thread.daemon = True
self.__thread.start()
def is_locked(self):
"""
Return True if db locked
"""
return self.__thread is not None and self.__thread.isAlive()
def stop(self):
"""
Stop scan
"""
self.__thread = None
#######################
# PRIVATE #
#######################
def __clean_charts(self):
"""
Clean charts in db
"""
track_ids = Lp().tracks.get_old_charts_track_ids(int(time()))
Lp().db.del_tracks(track_ids)
self.stop()
def __get_objects_for_uris(self, uris):
"""
Return all tracks/dirs for uris
@param uris as string
@return (track uri as [str], track dirs as [str],
ignore dirs as [str])
"""
tracks = []
ignore_dirs = []
track_dirs = list(uris)
walk_uris = list(uris)
while walk_uris:
uri = walk_uris.pop(0)
empty = True
try:
d = Lio.File.new_for_uri(uri)
infos = d.enumerate_children(
'standard::name,standard::type,standard::is-hidden',
Gio.FileQueryInfoFlags.NONE,
None)
except Exception as e:
print("CollectionScanner::__get_objects_for_uris():", e)
continue
for info in infos:
f = infos.get_child(info)
child_uri = f.get_uri()
empty = False
if info.get_is_hidden():
continue
elif info.get_file_type() == Gio.FileType.DIRECTORY:
track_dirs.append(child_uri)
walk_uris.append(child_uri)
else:
try:
f = Lio.File.new_for_uri(child_uri)
if is_pls(f):
pass
elif is_audio(f):
tracks.append(child_uri)
else:
debug("%s not detected as a music file" % uri)
except Exception as e:
print("CollectionScanner::"
"__get_objects_for_uris():", e)
# If a root uri is empty
# Ensure user is not doing something bad
if empty and uri in uris:
ignore_dirs.append(uri)
return (tracks, track_dirs, ignore_dirs)
def __update_progress(self, current, total):
"""
Update progress bar status
@param scanned items as int, total items as int
"""
Lp().window.progress.set_fraction(current / total, self)
def __finish(self):
"""
Notify from main thread when scan finished
"""
Lp().window.progress.set_fraction(1.0, self)
self.stop()
self.emit("scan-finished")
# Update max count value
Lp().albums.update_max_count()
if Lp().settings.get_value('artist-artwork'):
Lp().art.cache_artists_info()
def __scan(self, uris):
"""
Scan music collection for music files
@param uris as [string], uris to scan
@thread safe
"""
gst_message = None
if self.__history is None:
self.__history = History()
mtimes = Lp().tracks.get_mtimes()
(new_tracks, new_dirs, ignore_dirs) = self.__get_objects_for_uris(
uris)
orig_tracks = Lp().tracks.get_uris(ignore_dirs)
was_empty = len(orig_tracks) == 0
if ignore_dirs:
if Lp().notify is not None:
Lp().notify.send(_("Lollypop is detecting an empty folder."),
_("Check your music settings."))
count = len(new_tracks) + len(orig_tracks)
# Add monitors on dirs
if self.__inotify is not None:
for d in new_dirs:
self.__inotify.add_monitor(d)
with SqlCursor(Lp().db) as sql:
i = 0
# Look for new files/modified files
try:
to_add = []
for uri in new_tracks:
if self.__thread is None:
return
GLib.idle_add(self.__update_progress, i, count)
f = Lio.File.new_for_uri(uri)
info = f.query_info('time::modified',
Gio.FileQueryInfoFlags.NONE,
None)
mtime = int(info.get_attribute_as_string('time::modified'))
# If songs exists and mtime unchanged, continue,
# else rescan
if uri in orig_tracks:
orig_tracks.remove(uri)
i += 1
if mtime <= mtimes[uri]:
i += 1
continue
else:
self.__del_from_db(uri)
# On first scan, use modification time
# Else, use current time
if not was_empty:
mtime = int(time())
to_add.append((uri, mtime))
# Clean deleted files
# Now because we need to populate history
for uri in orig_tracks:
i += 1
GLib.idle_add(self.__update_progress, i, count)
if uri.startswith('file:'):
self.__del_from_db(uri)
# Add files to db
for (uri, mtime) in to_add:
try:
debug("Adding file: %s" % uri)
i += 1
GLib.idle_add(self.__update_progress, i, count)
self.__add2db(uri, mtime)
except GLib.GError as e:
print("CollectionScanner::__scan:", e, uri)
if e.message != gst_message:
gst_message = e.message
if Lp().notify is not None:
Lp().notify.send(gst_message, uri)
sql.commit()
except Exception as e:
print("CollectionScanner::__scan()", e)
GLib.idle_add(self.__finish)
del self.__history
self.__history = None
def __add2db(self, uri, mtime):
"""
Add new file to db with informations
@param uri as string
@param mtime as int
@return track id as int
"""
f = Lio.File.new_for_uri(uri)
debug("CollectionScanner::add2db(): Read tags")
info = self.get_info(uri)
tags = info.get_tags()
name = f.get_basename()
title = self.get_title(tags, name)
artists = self.get_artists(tags)
composers = self.get_composers(tags)
performers = self.get_performers(tags)
a_sortnames = self.get_artist_sortnames(tags)
aa_sortnames = self.get_album_artist_sortnames(tags)
album_artists = self.get_album_artist(tags)
album_name = self.get_album_name(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
discname = self.get_discname(tags)
tracknumber = self.get_tracknumber(tags, name)
year = self.get_original_year(tags)
if year is None:
year = self.get_year(tags)
duration = int(info.get_duration()/1000000000)
# If no artists tag, use album artist
if artists == '':
artists = album_artists
# if artists is always null, no album artists too,
# use composer/performer
if artists == '':
artists = performers
album_artists = composers
if artists == '':
artists = album_artists
if artists == '':
artists = _("Unknown")
debug("CollectionScanner::add2db(): Restore stats")
# Restore stats
(track_pop, track_rate, track_ltime, amtime,
loved, album_pop, album_rate) = self.__history.get(name, duration)
# If nothing in stats, use track mtime
if amtime == 0:
amtime = mtime
debug("CollectionScanner::add2db(): Add artists %s" % artists)
artist_ids = self.add_artists(artists, album_artists, a_sortnames)
debug("CollectionScanner::add2db(): "
"Add album artists %s" % album_artists)
album_artist_ids = self.add_album_artists(album_artists, aa_sortnames)
new_artist_ids = list(set(album_artist_ids) | set(artist_ids))
debug("CollectionScanner::add2db(): Add album: "
"%s, %s" % (album_name, album_artist_ids))
(album_id, new_album) = self.add_album(album_name, album_artist_ids,
uri, loved, album_pop,
album_rate, False)
genre_ids = self.add_genres(genres)
# Add track to db
debug("CollectionScanner::add2db(): Add track")
track_id = Lp().tracks.add(title, uri, duration,
tracknumber, discnumber, discname,
album_id, year, track_pop, track_rate,
track_ltime)
debug("CollectionScanner::add2db(): Update tracks")
self.update_track(track_id, artist_ids, genre_ids, mtime)
self.update_album(album_id, album_artist_ids, genre_ids, amtime, year)
if new_album:
with SqlCursor(Lp().db) as sql:
sql.commit()
for genre_id in genre_ids:
GLib.idle_add(self.emit, 'genre-updated', genre_id, True)
for artist_id in new_artist_ids:
GLib.idle_add(self.emit, 'artist-updated', artist_id, True)
return track_id
def __del_from_db(self, uri):
"""
Delete track from db
@param uri as str
"""
f = Lio.File.new_for_uri(uri)
name = f.get_basename()
track_id = Lp().tracks.get_id_by_uri(uri)
album_id = Lp().tracks.get_album_id(track_id)
genre_ids = Lp().tracks.get_genre_ids(track_id)
album_artist_ids = Lp().albums.get_artist_ids(album_id)
artist_ids = Lp().tracks.get_artist_ids(track_id)
popularity = Lp().tracks.get_popularity(track_id)
rate = Lp().tracks.get_rate(track_id)
ltime = Lp().tracks.get_ltime(track_id)
mtime = Lp().albums.get_mtime(album_id)
duration = Lp().tracks.get_duration(track_id)
album_popularity = Lp().albums.get_popularity(album_id)
album_rate = Lp().albums.get_rate(album_id)
loved = Lp().albums.get_loved(album_id)
uri = Lp().tracks.get_uri(track_id)
self.__history.add(name, duration, popularity, rate,
ltime, mtime, loved, album_popularity, album_rate)
Lp().tracks.remove(track_id)
Lp().tracks.clean(track_id)
deleted = Lp().albums.clean(album_id)
if deleted:
with SqlCursor(Lp().db) as sql:
sql.commit()
GLib.idle_add(self.emit, 'album-updated', album_id, True)
for artist_id in album_artist_ids + artist_ids:
Lp().artists.clean(artist_id)
GLib.idle_add(self.emit, 'artist-updated', artist_id, False)
for genre_id in genre_ids:
Lp().genres.clean(genre_id)
GLib.idle_add(self.emit, 'genre-updated', genre_id, False)
class CollectionScanner(GObject.GObject, ScannerTagReader):
"""
Scan user music collection
"""
__gsignals__ = {
'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
'artist-update': (GObject.SignalFlags.RUN_FIRST, None, (int, int)),
'genre-update': (GObject.SignalFlags.RUN_FIRST, None, (int,)),
'album-modified': (GObject.SignalFlags.RUN_FIRST, None, (int,))
}
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
ScannerTagReader.__init__(self)
self._inotify = None
if Lp.settings.get_value('auto-update'):
self._inotify = Inotify()
self._is_empty = True
self._in_thread = False
self._is_locked = False
self._progress = None
def update(self, progress):
"""
Update database
@param progress as progress bar
"""
# Keep track of on file with missing codecs
self._missing_codecs = None
self.init_discover()
self._progress = progress
paths = Lp.settings.get_music_paths()
if not paths:
return
if not self._in_thread:
if Lp.notify is not None:
Lp.notify.send(_("Your music is updating"))
if self._progress is not None:
self._progress.show()
self._in_thread = True
self._is_locked = True
start_new_thread(self._scan, (paths,))
def is_locked(self):
"""
Return True if db locked
"""
return self._is_locked
def stop(self):
"""
Stop scan
"""
if self._progress is not None:
self._progress.hide()
self._progress.set_fraction(0.0)
self._progress = None
self._in_thread = False
#######################
# PRIVATE #
#######################
def _get_objects_for_paths(self, paths):
"""
Return all tracks/dirs for paths
@param paths as string
@return ([tracks path], [dirs path], track count)
"""
tracks = []
track_dirs = list(paths)
count = 0
for path in paths:
for root, dirs, files in os.walk(path):
# Add dirs
for d in dirs:
track_dirs.append(os.path.join(root, d))
# Add files
for name in files:
filepath = os.path.join(root, name)
try:
f = Gio.File.new_for_path(filepath)
if is_pls(f):
pass
elif is_audio(f):
tracks.append(filepath)
count += 1
else:
debug("%s not detected as a music file" % filepath)
except Exception as e:
print("CollectionScanner::_get_objects_for_paths: %s"
% e)
return (tracks, track_dirs, count)
def _update_progress(self, current, total):
"""
Update progress bar status
@param scanned items as int, total items as int
"""
if self._progress is not None:
self._progress.set_fraction(current/total)
def _finish(self):
"""
Notify from main thread when scan finished
"""
if self._progress is not None:
self._progress.hide()
self._progress.set_fraction(0.0)
self._progress = None
self._in_thread = False
self._is_locked = False
self.emit("scan-finished")
if self._missing_codecs is not None:
Lp.player.load_external(GLib.filename_to_uri(self._missing_codecs))
Lp.player.play_first_external()
def _scan(self, paths):
"""
Scan music collection for music files
@param paths as [string], paths to scan
@thread safe
"""
sql = Lp.db.get_cursor()
mtimes = Lp.tracks.get_mtimes(sql)
orig_tracks = Lp.tracks.get_paths(sql)
self._is_empty = len(orig_tracks) == 0
# Add monitors on dirs
(new_tracks, new_dirs, count) = self._get_objects_for_paths(paths)
if self._inotify is not None:
for d in new_dirs:
self._inotify.add_monitor(d)
i = 0
for filepath in new_tracks:
if not self._in_thread:
sql.close()
self._is_locked = False
return
GLib.idle_add(self._update_progress, i, count)
try:
mtime = int(os.path.getmtime(filepath))
if filepath not in orig_tracks:
try:
infos = self.get_infos(filepath)
debug("Adding file: %s" % filepath)
self._add2db(filepath, mtime, infos, sql)
except Exception as e:
debug("Error scanning: %s" % filepath)
string = "%s" % e
if string.startswith('gst-core-error-quark'):
self._missing_codecs = filepath
else:
# Update tags by removing song and readd it
if mtime != mtimes[filepath]:
self._del_from_db(filepath, sql)
infos = self.get_infos(filepath)
if infos is not None:
debug("Adding file: %s" % filepath)
self._add2db(filepath, mtime, infos, sql)
else:
print("Can't get infos for ", filepath)
orig_tracks.remove(filepath)
except Exception as e:
print(ascii(filepath))
print("CollectionScanner::_scan(): %s" % e)
i += 1
# Clean deleted files
if i > 0:
for filepath in orig_tracks:
self._del_from_db(filepath, sql)
self._restore_albums_popularity(sql)
self._restore_albums_mtime(sql)
self._restore_tracks_popularity(sql)
self._restore_tracks_ltime(sql)
sql.commit()
sql.close()
GLib.idle_add(self._finish)
def _add2db(self, filepath, mtime, infos, sql):
"""
Add new file to db with informations
@param filepath as string
@param file modification time as int
@param infos as GstPbutils.DiscovererInfo
@param sql as sqlite cursor
@return track id as int
"""
tags = infos.get_tags()
title = self.get_title(tags, filepath)
artists = self.get_artists(tags)
album_artist = self.get_album_artist(tags)
album_name = self.get_album_name(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
tracknumber = self.get_tracknumber(tags)
year = self.get_year(tags)
length = infos.get_duration()/1000000000
(artist_ids, new_artist_ids) = self.add_artists(artists,
album_artist,
sql)
(album_artist_id, new) = self.add_album_artist(album_artist,
sql)
if new:
new_artist_ids.append(album_artist_id)
noaartist = False
if album_artist_id is None:
album_artist_id = artist_ids[0]
noaartist = True
album_id = self.add_album(album_name, album_artist_id, noaartist,
filepath, sql)
(genre_ids, new_genre_ids) = self.add_genres(genres, album_id, sql)
# Add track to db
Lp.tracks.add(title, filepath, length,
tracknumber, discnumber,
album_id, year, 0, 0, mtime, sql)
self.update_year(album_id, sql)
track_id = Lp.tracks.get_id_by_path(filepath, sql)
self.update_track(track_id, artist_ids, genre_ids, sql)
# Notify about new artists/genres
if new_genre_ids or new_artist_ids:
sql.commit()
for genre_id in new_genre_ids:
GLib.idle_add(self.emit, 'genre-update', genre_id)
for artist_id in new_artist_ids:
GLib.idle_add(self.emit, 'artist-update', artist_id, album_id)
return track_id
def _del_from_db(self, filepath, sql):
"""
Delete track from db
@param filepath as string
@param sql as sqlite cursor
"""
track_id = Lp.tracks.get_id_by_path(filepath, sql)
album_id = Lp.tracks.get_album_id(track_id, sql)
genre_ids = Lp.tracks.get_genre_ids(track_id, sql)
album_artist_id = Lp.albums.get_artist_id(album_id, sql)
artist_ids = Lp.tracks.get_artist_ids(track_id, sql)
Lp.tracks.remove(filepath, sql)
Lp.tracks.clean(track_id, sql)
modified = Lp.albums.clean(album_id, sql)
if modified:
self.emit('album-modified', album_id)
for artist_id in [album_artist_id] + artist_ids:
Lp.artists.clean(artist_id, sql)
for genre_id in genre_ids:
Lp.genres.clean(genre_id, sql)
def _restore_albums_popularity(self, sql):
"""
Restore albums popularty
"""
popularities = Lp.db.get_albums_popularity()
result = sql.execute("SELECT albums.name, artists.name, albums.rowid\
FROM albums, artists\
WHERE artists.rowid == albums.artist_id")
for row in result:
string = "%s_%s" % (row[0], row[1])
if string in popularities:
Lp.albums.set_popularity(row[2],
popularities[string], sql)
def _restore_albums_mtime(self, sql):
"""
Restore albums mtime
"""
mtimes = Lp.db.get_albums_mtime()
result = sql.execute("SELECT albums.name, artists.name, albums.rowid\
FROM albums, artists\
WHERE artists.rowid == albums.artist_id")
for row in result:
string = "%s_%s" % (row[0], row[1])
if string in mtimes:
Lp.albums.set_mtime(row[2],
mtimes[string], sql)
def _restore_tracks_popularity(self, sql):
"""
Restore tracks popularty
"""
popularities = Lp.db.get_tracks_popularity()
result = sql.execute("SELECT tracks.name, artists.name, tracks.rowid\
FROM tracks, artists, track_artists\
WHERE artists.rowid == track_artists.artist_id\
AND track_artists.track_id == tracks.rowid")
for row in result:
string = "%s_%s" % (row[0], row[1])
if string in popularities:
Lp.tracks.set_popularity(row[2],
popularities[string], sql)
def _restore_tracks_ltime(self, sql):
"""
Restore tracks ltime
"""
ltimes = Lp.db.get_tracks_ltime()
result = sql.execute("SELECT tracks.name, artists.name, tracks.rowid\
FROM tracks, artists, track_artists\
WHERE artists.rowid == track_artists.artist_id\
AND track_artists.track_id == tracks.rowid")
for row in result:
string = "%s_%s" % (row[0], row[1])
if string in ltimes:
Lp.tracks.set_ltime(row[2],
ltimes[string], sql)
class CollectionScanner(GObject.GObject, TagReader):
"""
Scan user music collection
"""
__gsignals__ = {
'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
'artist-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
'genre-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
'album-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool))
}
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
TagReader.__init__(self)
self.__thread = None
self.__history = None
if Lp().settings.get_value('auto-update'):
self.__inotify = Inotify()
else:
self.__inotify = None
def update(self):
"""
Update database
"""
if not self.is_locked():
paths = Lp().settings.get_music_paths()
if not paths:
return
Lp().window.progress.add(self)
Lp().window.progress.set_fraction(0.0, self)
if Lp().notify is not None:
Lp().notify.send(_("Your music is updating"))
self.__thread = Thread(target=self.__scan, args=(paths,))
self.__thread.daemon = True
self.__thread.start()
def is_locked(self):
"""
Return True if db locked
"""
return self.__thread is not None and self.__thread.isAlive()
def stop(self):
"""
Stop scan
"""
self.__thread = None
#######################
# PRIVATE #
#######################
def __get_objects_for_paths(self, paths):
"""
Return all tracks/dirs for paths
@param paths as string
@return (track path as [str], track dirs as [str],
ignore dirs as [str])
"""
tracks = []
ignore_dirs = []
track_dirs = list(paths)
for path in paths:
tracks_for_path = []
for root, dirs, files in os.walk(path, followlinks=True):
# Add dirs
for d in dirs:
track_dirs.append(os.path.join(root, d))
# Add files
for name in files:
path = os.path.join(root, name)
uri = GLib.filename_to_uri(path)
try:
f = Gio.File.new_for_uri(uri)
if is_pls(f):
pass
elif is_audio(f):
tracks_for_path.append(uri)
else:
debug("%s not detected as a music file" % uri)
except Exception as e:
print("CollectionScanner::__get_objects_for_paths: %s"
% e)
# If a path is empty
# Ensure user is not doing something bad
if not tracks_for_path:
ignore_dirs.append(GLib.filename_to_uri(path))
tracks += tracks_for_path
return (tracks, track_dirs, ignore_dirs)
def __update_progress(self, current, total):
"""
Update progress bar status
@param scanned items as int, total items as int
"""
Lp().window.progress.set_fraction(current / total, self)
def __finish(self):
"""
Notify from main thread when scan finished
"""
Lp().window.progress.set_fraction(1.0, self)
self.stop()
self.emit("scan-finished")
if Lp().settings.get_value('artist-artwork'):
Lp().art.cache_artists_info()
def __scan(self, paths):
"""
Scan music collection for music files
@param paths as [string], paths to scan
@thread safe
"""
gst_message = None
if self.__history is None:
self.__history = History()
mtimes = Lp().tracks.get_mtimes()
(new_tracks, new_dirs, ignore_dirs) = self.__get_objects_for_paths(
paths)
orig_tracks = Lp().tracks.get_uris(ignore_dirs)
was_empty = len(orig_tracks) == 0
if ignore_dirs:
if Lp().notify is not None:
Lp().notify.send(_("Lollypop is detecting an empty folder."),
_("Check your music settings."))
count = len(new_tracks) + len(orig_tracks)
# Add monitors on dirs
if self.__inotify is not None:
for d in new_dirs:
self.__inotify.add_monitor(d)
with SqlCursor(Lp().db) as sql:
i = 0
# Get deleted files
deleted = list(orig_tracks)
for uri in new_tracks:
if uri in deleted:
deleted.remove(uri)
# Clean deleted files
for uri in deleted:
i += 1
GLib.idle_add(self.__update_progress, i, count)
if uri.startswith('file:'):
self.__del_from_db(uri)
# Look for new files/modified files
for uri in new_tracks:
if self.__thread is None:
return
GLib.idle_add(self.__update_progress, i, count)
try:
f = Gio.File.new_for_uri(uri)
info = f.query_info('time::modified',
Gio.FileQueryInfoFlags.NONE,
None)
mtime = int(info.get_attribute_as_string('time::modified'))
# If songs exists and mtime unchanged, continue,
# else rescan
if uri in orig_tracks:
orig_tracks.remove(uri)
i += 1
if mtime <= mtimes[uri]:
i += 1
continue
else:
self.__del_from_db(uri)
info = self.get_info(uri)
# On first scan, use modification time
# Else, use current time
if not was_empty:
mtime = int(time())
debug("Adding file: %s" % uri)
self.__add2db(uri, info, mtime)
except GLib.GError as e:
print(e, uri)
if e.message != gst_message:
gst_message = e.message
if Lp().notify is not None:
Lp().notify.send(gst_message)
except Exception as e:
print("CollectionScanner::__scan()", e)
i += 1
sql.commit()
GLib.idle_add(self.__finish)
del self.__history
self.__history = None
def __add2db(self, uri, info, mtime):
"""
Add new file to db with informations
@param uri as string
@param info as GstPbutils.DiscovererInfo
@param mtime as int
@return track id as int
"""
debug("CollectionScanner::add2db(): Read tags")
path = GLib.filename_from_uri(uri)[0]
tags = info.get_tags()
title = self.get_title(tags, path)
artists = self.get_artists(tags)
composers = self.get_composers(tags)
performers = self.get_performers(tags)
a_sortnames = self.get_artist_sortnames(tags)
aa_sortnames = self.get_album_artist_sortnames(tags)
album_artists = self.get_album_artist(tags)
album_name = self.get_album_name(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
discname = self.get_discname(tags)
name = GLib.path_get_basename(path)
tracknumber = self.get_tracknumber(tags, name)
year = self.get_year(tags)
duration = int(info.get_duration()/1000000000)
# If no artists tag, use album artist
if artists == '':
artists = album_artists
# if artists is always null, no album artists too,
# use composer/performer
if artists == '':
artists = performers
album_artists = composers
if artists == '':
artists = album_artists
if artists == '':
artists = _("Unknown")
debug("CollectionScanner::add2db(): Restore stats")
# Restore stats
(track_pop, track_ltime, amtime, album_pop) = self.__history.get(
name, duration)
# If nothing in stats, set mtime
if amtime == 0:
amtime = mtime
debug("CollectionScanner::add2db(): Add artists %s" % artists)
artist_ids = self.add_artists(artists, album_artists, a_sortnames)
debug("CollectionScanner::add2db(): "
"Add album artists %s" % album_artists)
album_artist_ids = self.add_album_artists(album_artists, aa_sortnames)
new_artist_ids = list(set(album_artist_ids) | set(artist_ids))
debug("CollectionScanner::add2db(): Add album: "
"%s, %s" % (album_name, album_artist_ids))
(album_id, new_album) = self.add_album(album_name, album_artist_ids,
path, album_pop, amtime)
genre_ids = self.add_genres(genres, album_id)
# Add track to db
debug("CollectionScanner::add2db(): Add track")
track_id = Lp().tracks.add(title, uri, duration,
tracknumber, discnumber, discname,
album_id, year, track_pop,
track_ltime, mtime)
debug("CollectionScanner::add2db(): Update tracks")
self.update_track(track_id, artist_ids, genre_ids)
self.update_album(album_id, album_artist_ids, genre_ids, year)
if new_album:
with SqlCursor(Lp().db) as sql:
sql.commit()
for genre_id in genre_ids:
GLib.idle_add(self.emit, 'genre-updated', genre_id, True)
for artist_id in new_artist_ids:
GLib.idle_add(self.emit, 'artist-updated', artist_id, True)
return track_id
def __del_from_db(self, uri):
"""
Delete track from db
@param uri as str
"""
path = GLib.filename_from_uri(uri)[0]
name = GLib.path_get_basename(path)
track_id = Lp().tracks.get_id_by_uri(uri)
album_id = Lp().tracks.get_album_id(track_id)
genre_ids = Lp().tracks.get_genre_ids(track_id)
album_artist_ids = Lp().albums.get_artist_ids(album_id)
artist_ids = Lp().tracks.get_artist_ids(track_id)
popularity = Lp().tracks.get_popularity(track_id)
ltime = Lp().tracks.get_ltime(track_id)
mtime = Lp().albums.get_mtime(album_id)
duration = Lp().tracks.get_duration(track_id)
album_popularity = Lp().albums.get_popularity(album_id)
self.__history.add(name, duration, popularity,
ltime, mtime, album_popularity)
Lp().tracks.remove(track_id)
Lp().tracks.clean(track_id)
deleted = Lp().albums.clean(album_id)
if deleted:
with SqlCursor(Lp().db) as sql:
sql.commit()
GLib.idle_add(self.emit, 'album-updated', album_id, True)
for artist_id in album_artist_ids + artist_ids:
Lp().artists.clean(artist_id)
GLib.idle_add(self.emit, 'artist-updated', artist_id, False)
for genre_id in genre_ids:
Lp().genres.clean(genre_id)
GLib.idle_add(self.emit, 'genre-updated', genre_id, False)
class CollectionScanner(GObject.GObject, TagReader):
"""
Scan user music collection
"""
__gsignals__ = {
"scan-finished": (GObject.SignalFlags.RUN_FIRST, None, (bool, )),
"updated":
(GObject.SignalFlags.RUN_FIRST, None, (GObject.TYPE_PYOBJECT, int))
}
def __init__(self):
"""
Init collection scanner
"""
GObject.GObject.__init__(self)
self.__thread = None
self.__tags = {}
self.__items = []
self.__pending_new_artist_ids = []
self.__history = History()
self.__progress_total = 1
self.__progress_count = 0
self.__progress_fraction = 0
self.__disable_compilations = not App().settings.get_value(
"show-compilations")
if App().settings.get_value("auto-update"):
self.__inotify = Inotify()
else:
self.__inotify = None
App().albums.update_max_count()
def update(self, scan_type, uris=[]):
"""
Update database
@param scan_type as ScanType
@param uris as [str]
"""
self.__disable_compilations = not App().settings.get_value(
"show-compilations")
App().lookup_action("update_db").set_enabled(False)
# Stop previous scan
if self.is_locked() and scan_type != ScanType.EXTERNAL:
self.stop()
GLib.timeout_add(250, self.update, scan_type, uris)
elif App().ws_director.collection_ws is not None and\
not App().ws_director.collection_ws.stop():
GLib.timeout_add(250, self.update, scan_type, uris)
else:
if scan_type == ScanType.FULL:
uris = App().settings.get_music_uris()
if not uris:
return
# Register to progressbar
if scan_type != ScanType.EXTERNAL:
App().window.container.progress.add(self)
App().window.container.progress.set_fraction(0, self)
Logger.info("Scan started")
# Launch scan in a separate thread
self.__thread = App().task_helper.run(self.__scan, scan_type, uris)
def save_album(
self,
item,
):
"""
Add album to DB
@param item as CollectionItem
"""
Logger.debug("CollectionScanner::save_album(): "
"Add album artists %s" % item.album_artists)
(item.new_album_artist_ids,
item.album_artist_ids) = self.add_artists(item.album_artists,
item.aa_sortnames,
item.mb_album_artist_id)
# We handle artists already created by any previous save_track()
for artist_id in item.album_artist_ids:
if artist_id in self.__pending_new_artist_ids:
item.new_album_artist_ids.append(artist_id)
self.__pending_new_artist_ids.remove(artist_id)
item.lp_album_id = get_lollypop_album_id(item.album_name,
item.album_artists)
Logger.debug("CollectionScanner::save_track(): Add album: "
"%s, %s" % (item.album_name, item.album_artist_ids))
(item.new_album, item.album_id) = self.add_album(
item.album_name, item.mb_album_id, item.lp_album_id,
item.album_artist_ids, item.uri, item.album_loved, item.album_pop,
item.album_rate, item.album_synced, item.album_mtime,
item.storage_type)
if item.year is not None:
App().albums.set_year(item.album_id, item.year)
App().albums.set_timestamp(item.album_id, item.timestamp)
def save_track(self, item):
"""
Add track to DB
@param item as CollectionItem
"""
Logger.debug("CollectionScanner::save_track(): Add artists %s" %
item.artists)
(item.new_artist_ids,
item.artist_ids) = self.add_artists(item.artists, item.a_sortnames,
item.mb_artist_id)
self.__pending_new_artist_ids += item.new_artist_ids
missing_artist_ids = list(
set(item.album_artist_ids) - set(item.artist_ids))
# Special case for broken tags
# If all artist album tags are missing
# Can't do more because don't want to break split album behaviour
if len(missing_artist_ids) == len(item.album_artist_ids):
item.artist_ids += missing_artist_ids
if item.genres is None:
(item.new_genre_ids, item.genre_ids) = ([], [Type.WEB])
else:
(item.new_genre_ids, item.genre_ids) = self.add_genres(item.genres)
item.lp_track_id = get_lollypop_track_id(item.track_name, item.artists,
item.album_name)
# Add track to db
Logger.debug("CollectionScanner::save_track(): Add track")
item.track_id = App().tracks.add(
item.track_name, item.uri, item.duration, item.tracknumber,
item.discnumber, item.discname, item.album_id, item.year,
item.timestamp, item.track_pop, item.track_rate, item.track_loved,
item.track_ltime, item.track_mtime, item.mb_track_id,
item.lp_track_id, item.bpm, item.storage_type)
Logger.debug("CollectionScanner::save_track(): Update track")
self.update_track(item)
Logger.debug("CollectionScanner::save_track(): Update album")
self.update_album(item)
def update_album(self, item):
"""
Update album artists based on album-artist and artist tags
This code auto handle compilations: empty "album artist" with
different artists
@param item as CollectionItem
"""
if item.album_artist_ids:
App().albums.set_artist_ids(item.album_id, item.album_artist_ids)
# Set artist ids based on content
else:
if item.compilation:
new_album_artist_ids = [Type.COMPILATIONS]
else:
new_album_artist_ids = App().albums.calculate_artist_ids(
item.album_id, self.__disable_compilations)
App().albums.set_artist_ids(item.album_id, new_album_artist_ids)
# We handle artists already created by any previous save_track()
item.new_album_artist_ids = []
for artist_id in new_album_artist_ids:
if artist_id in self.__pending_new_artist_ids:
item.new_album_artist_ids.append(artist_id)
self.__pending_new_artist_ids.remove(artist_id)
# Update lp_album_id
lp_album_id = get_lollypop_album_id(item.album_name,
item.album_artists)
if lp_album_id != item.lp_album_id:
App().art.move_artwork(item.lp_album_id, lp_album_id)
App().albums.set_lp_album_id(item.album_id, lp_album_id)
item.lp_album_id = lp_album_id
# Update album genres
for genre_id in item.genre_ids:
App().albums.add_genre(item.album_id, genre_id)
# Update year based on tracks
year = App().tracks.get_year_for_album(item.album_id)
if year is not None:
App().albums.set_year(item.album_id, year)
timestamp = App().tracks.get_timestamp_for_album(item.album_id)
App().albums.set_timestamp(item.album_id, timestamp)
App().cache.clear_durations(item.album_id)
def update_track(self, item):
"""
Set track artists/genres
@param item as CollectionItem
"""
# Set artists/genres for track
for artist_id in item.artist_ids:
App().tracks.add_artist(item.track_id, artist_id)
for genre_id in item.genre_ids:
App().tracks.add_genre(item.track_id, genre_id)
def del_from_db(self, uri, backup):
"""
Delete track from db
@param uri as str
@param backup as bool
@return (popularity, ltime, mtime,
loved album, album_popularity)
"""
try:
track_id = App().tracks.get_id_by_uri(uri)
duration = App().tracks.get_duration(track_id)
album_id = App().tracks.get_album_id(track_id)
album_artist_ids = App().albums.get_artist_ids(album_id)
artist_ids = App().tracks.get_artist_ids(track_id)
track_pop = App().tracks.get_popularity(track_id)
track_rate = App().tracks.get_rate(track_id)
track_ltime = App().tracks.get_ltime(track_id)
album_mtime = App().tracks.get_mtime(track_id)
track_loved = App().tracks.get_loved(track_id)
album_pop = App().albums.get_popularity(album_id)
album_rate = App().albums.get_rate(album_id)
album_loved = App().albums.get_loved(album_id)
album_synced = App().albums.get_synced(album_id)
if backup:
f = Gio.File.new_for_uri(uri)
name = f.get_basename()
self.__history.add(name, duration, track_pop, track_rate,
track_ltime, album_mtime, track_loved,
album_loved, album_pop, album_rate,
album_synced)
App().tracks.remove(track_id)
genre_ids = App().tracks.get_genre_ids(track_id)
App().albums.clean()
App().genres.clean()
App().artists.clean()
App().cache.clear_durations(album_id)
SqlCursor.commit(App().db)
item = CollectionItem(album_id=album_id)
if not App().albums.get_name(album_id):
item.artist_ids = []
for artist_id in album_artist_ids + artist_ids:
if not App().artists.get_name(artist_id):
item.artist_ids.append(artist_id)
item.genre_ids = []
for genre_id in genre_ids:
if not App().genres.get_name(genre_id):
item.genre_ids.append(genre_id)
emit_signal(self, "updated", item, ScanUpdate.REMOVED)
else:
# Force genre for album
genre_ids = App().tracks.get_album_genre_ids(album_id)
App().albums.set_genre_ids(album_id, genre_ids)
emit_signal(self, "updated", item, ScanUpdate.MODIFIED)
return (track_pop, track_rate, track_ltime, album_mtime,
track_loved, album_loved, album_pop, album_rate)
except Exception as e:
Logger.error("CollectionScanner::del_from_db: %s" % e)
def is_locked(self):
"""
True if db locked
@return bool
"""
return self.__thread is not None and self.__thread.is_alive()
def stop(self):
"""
Stop scan
"""
self.__thread = None
def reset_database(self):
"""
Reset database
"""
from lollypop.app_notification import AppNotification
App().window.container.progress.add(self)
App().window.container.progress.set_fraction(0, self)
self.__progress_fraction = 0
notification = AppNotification(_("Resetting database"), [], [])
notification.show()
App().window.container.add_overlay(notification)
notification.set_reveal_child(True)
App().task_helper.run(self.__reset_database)
@property
def inotify(self):
"""
Get Inotify object
@return Inotify
"""
return self.__inotify
#######################
# PRIVATE #
#######################
def __reset_database(self):
"""
Reset database
"""
def update_ui():
App().window.container.go_home()
App().scanner.update(ScanType.FULL)
App().player.stop()
if App().ws_director.collection_ws is not None:
App().ws_director.collection_ws.stop()
uris = App().tracks.get_uris()
i = 0
SqlCursor.add(App().db)
SqlCursor.add(self.__history)
count = len(uris)
for uri in uris:
self.del_from_db(uri, True)
self.__update_progress(i, count, 0.01)
i += 1
App().tracks.del_persistent(False)
App().tracks.clean(False)
App().albums.clean(False)
App().artists.clean(False)
App().genres.clean(False)
App().cache.clear_table("duration")
SqlCursor.commit(App().db)
SqlCursor.remove(App().db)
SqlCursor.commit(self.__history)
SqlCursor.remove(self.__history)
GLib.idle_add(update_ui)
def __update_progress(self, current, total, allowed_diff):
"""
Update progress bar status
@param current as int
@param total as int
@param allowed_diff as float => allows to prevent
main loop flooding
"""
new_fraction = current / total
if new_fraction > self.__progress_fraction + allowed_diff:
self.__progress_fraction = new_fraction
GLib.idle_add(App().window.container.progress.set_fraction,
new_fraction, self)
def __finish(self, items):
"""
Notify from main thread when scan finished
@param items as [CollectionItem]
"""
track_ids = [item.track_id for item in items]
self.__thread = None
Logger.info("Scan finished")
App().lookup_action("update_db").set_enabled(True)
App().window.container.progress.set_fraction(1.0, self)
self.stop()
emit_signal(self, "scan-finished", track_ids)
# Update max count value
App().albums.update_max_count()
# Update featuring
App().artists.update_featuring()
if App().ws_director.collection_ws is not None:
App().ws_director.collection_ws.start()
def __add_monitor(self, dirs):
"""
Monitor any change in a list of directory
@param dirs as str or list of directory to be monitored
"""
if self.__inotify is None:
return
# Add monitors on dirs
for d in dirs:
# Handle a stop request
if self.__thread is None:
break
if d.startswith("file://"):
self.__inotify.add_monitor(d)
@profile
def __get_objects_for_uris(self, scan_type, uris):
"""
Get all tracks and dirs in uris
@param scan_type as ScanType
@param uris as string
@return ([(int, str)], [str], [str])
([(mtime, file)], [dir], [stream])
"""
files = []
dirs = []
streams = []
walk_uris = []
# Check collection exists
for uri in uris:
parsed = urlparse(uri)
if parsed.scheme in ["http", "https"]:
streams.append(uri)
else:
f = Gio.File.new_for_uri(uri)
if f.query_exists():
walk_uris.append(uri)
else:
return ([], [], [])
while walk_uris:
uri = walk_uris.pop(0)
try:
# Directly add files, walk through directories
f = Gio.File.new_for_uri(uri)
info = f.query_info(SCAN_QUERY_INFO,
Gio.FileQueryInfoFlags.NONE, None)
if info.get_file_type() == Gio.FileType.DIRECTORY:
dirs.append(uri)
infos = f.enumerate_children(SCAN_QUERY_INFO,
Gio.FileQueryInfoFlags.NONE,
None)
for info in infos:
f = infos.get_child(info)
child_uri = f.get_uri()
if info.get_is_hidden():
continue
# User do not want internal symlinks
elif info.get_is_symlink() and\
App().settings.get_value("ignore-symlinks"):
continue
walk_uris.append(child_uri)
infos.close(None)
# Only happens if files passed as args
else:
mtime = get_mtime(info)
files.append((mtime, uri))
except Exception as e:
Logger.error(
"CollectionScanner::__get_objects_for_uris(): %s" % e)
files.sort(reverse=True)
return (files, dirs, streams)
@profile
def __scan(self, scan_type, uris):
"""
Scan music collection for music files
@param scan_type as ScanType
@param uris as [str]
@thread safe
"""
try:
SqlCursor.add(App().db)
App().art.clean_rounded()
(files, dirs,
streams) = self.__get_objects_for_uris(scan_type, uris)
if not files:
App().notify.send("Lollypop",
_("Scan disabled, missing collection"))
return
if scan_type == ScanType.NEW_FILES:
db_uris = App().tracks.get_uris(uris)
else:
db_uris = App().tracks.get_uris()
# Get mtime of all tracks to detect which has to be updated
db_mtimes = App().tracks.get_mtimes()
# * 2 => Scan + Save
self.__progress_total = len(files) * 2 + len(streams)
self.__progress_count = 0
self.__progress_fraction = 0
# Min: 1 thread, Max: 5 threads
count = max(1, min(5, cpu_count() // 2))
split_files = split_list(files, count)
self.__tags = {}
self.__pending_new_artist_ids = []
threads = []
for files in split_files:
thread = App().task_helper.run(self.__scan_files, files,
db_mtimes, scan_type)
threads.append(thread)
if scan_type == ScanType.EXTERNAL:
storage_type = StorageType.EXTERNAL
else:
storage_type = StorageType.COLLECTION
# Start getting files and populating DB
self.__items = []
i = 0
while threads:
thread = threads[i]
if not thread.is_alive():
threads.remove(thread)
self.__items += self.__save_in_db(storage_type)
if i >= len(threads) - 1:
i = 0
else:
i += 1
# Add streams to DB, only happening on command line/m3u files
self.__items += self.__save_streams_in_db(streams, storage_type)
self.__remove_old_tracks(db_uris, scan_type)
if scan_type == ScanType.EXTERNAL:
albums = tracks_to_albums(
[Track(item.track_id) for item in self.__items])
App().player.play_albums(albums)
else:
self.__add_monitor(dirs)
GLib.idle_add(self.__finish, self.__items)
self.__tags = {}
self.__items = []
self.__pending_new_artist_ids = []
except Exception as e:
Logger.warning("CollectionScanner::__scan(): %s", e)
SqlCursor.remove(App().db)
def __scan_to_handle(self, uri):
"""
Check if file has to be handle by scanner
@param f as Gio.File
@return bool
"""
try:
file_type = get_file_type(uri)
# Get file type using Gio (slower)
if file_type == FileType.UNKNOWN:
f = Gio.File.new_for_uri(uri)
info = f.query_info(FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE,
Gio.FileQueryInfoFlags.NONE)
if is_pls(info):
file_type = FileType.PLS
elif is_audio(info):
file_type = FileType.AUDIO
if file_type == FileType.PLS:
Logger.debug("Importing playlist %s" % uri)
if App().settings.get_value("import-playlists"):
App().playlists.import_tracks(uri)
elif file_type == FileType.AUDIO:
Logger.debug("Importing audio %s" % uri)
return True
except Exception as e:
Logger.error("CollectionScanner::__scan_to_handle(): %s" % e)
return False
def __scan_files(self, files, db_mtimes, scan_type):
"""
Scan music collection for new audio files
@param files as [str]
@param db_mtimes as {}
@param scan_type as ScanType
@thread safe
"""
discoverer = Discoverer()
try:
# Scan new files
for (mtime, uri) in files:
# Handle a stop request
if self.__thread is None and scan_type != ScanType.EXTERNAL:
raise Exception("cancelled")
try:
if not self.__scan_to_handle(uri):
self.__progress_count += 2
continue
db_mtime = db_mtimes.get(uri, 0)
if mtime > db_mtime:
# Do not use mtime if not intial scan
if db_mtimes:
mtime = int(time())
self.__tags[uri] = self.__get_tags(
discoverer, uri, mtime)
self.__progress_count += 1
self.__update_progress(self.__progress_count,
self.__progress_total, 0.001)
else:
# We want to play files, so put them in items
if scan_type == ScanType.EXTERNAL:
track_id = App().tracks.get_id_by_uri(uri)
item = CollectionItem(track_id=track_id)
self.__items.append(item)
self.__progress_count += 2
self.__update_progress(self.__progress_count,
self.__progress_total, 0.1)
except Exception as e:
Logger.error("Scanning file: %s, %s" % (uri, e))
except Exception as e:
Logger.warning("CollectionScanner::__scan_files(): % s" % e)
def __save_in_db(self, storage_type):
"""
Save current tags into DB
@param storage_type as StorageType
@return [CollectionItem]
"""
items = []
notify_index = 0
previous_album_id = None
for uri in list(self.__tags.keys()):
# Handle a stop request
if self.__thread is None:
raise Exception("cancelled")
Logger.debug("Adding file: %s" % uri)
tags = self.__tags[uri]
item = self.__add2db(uri, *tags, storage_type)
items.append(item)
self.__progress_count += 1
self.__update_progress(self.__progress_count,
self.__progress_total, 0.001)
if previous_album_id != item.album_id:
self.__notify_ui(items[notify_index:])
notify_index = len(items)
previous_album_id = item.album_id
del self.__tags[uri]
# Handle a stop request
if self.__thread is None:
raise Exception("cancelled")
self.__notify_ui(items)
return items
def __save_streams_in_db(self, streams, storage_type):
"""
Save http stream to DB
@param streams as [str]
@param storage_type as StorageType
@return [CollectionItem]
"""
items = []
for uri in streams:
parsed = urlparse(uri)
item = self.__add2db(uri, parsed.path, parsed.netloc, None, "", "",
parsed.netloc, parsed.netloc, "", False, 0,
False, 0, 0, 0, None, 0, "", "", "", "", 1, 0,
0, 0, 0, 0, False, 0, False, storage_type)
items.append(item)
self.__progress_count += 1
return items
def __notify_ui(self, items):
"""
Notify UI based on current items
@param items as [CollectionItem]
"""
SqlCursor.commit(App().db)
for item in items:
if item.new_album:
emit_signal(self, "updated", item, ScanUpdate.ADDED)
else:
emit_signal(self, "updated", item, ScanUpdate.MODIFIED)
def __remove_old_tracks(self, uris, scan_type):
"""
Remove non existent tracks from DB
@param scan_type as ScanType
"""
if scan_type != ScanType.EXTERNAL and self.__thread is not None:
# We need to check files are always in collections
if scan_type == ScanType.FULL:
collections = App().settings.get_music_uris()
else:
collections = None
for uri in uris:
# Handle a stop request
if self.__thread is None:
raise Exception("cancelled")
in_collection = True
if collections is not None:
in_collection = False
for collection in collections:
if collection in uri:
in_collection = True
break
f = Gio.File.new_for_uri(uri)
if not in_collection:
Logger.warning(
"Removed, not in collection anymore: %s -> %s", uri,
collections)
self.del_from_db(uri, True)
elif not f.query_exists():
Logger.warning("Removed, file has been deleted: %s", uri)
self.del_from_db(uri, True)
def __get_tags(self, discoverer, uri, track_mtime):
"""
Read track tags
@param discoverer as Discoverer
@param uri as string
@param track_mtime as int
@return ()
"""
f = Gio.File.new_for_uri(uri)
info = discoverer.get_info(uri)
tags = info.get_tags()
name = f.get_basename()
duration = int(info.get_duration() / 1000000)
Logger.debug("CollectionScanner::add2db(): Restore stats")
# Restore stats
track_id = App().tracks.get_id_by_uri(uri)
if track_id is None:
track_id = App().tracks.get_id_by_basename_duration(name, duration)
if track_id is None:
(track_pop, track_rate, track_ltime, album_mtime, track_loved,
album_loved, album_pop, album_rate,
album_synced) = self.__history.get(name, duration)
# Delete track and restore from it
else:
(track_pop, track_rate, track_ltime, album_mtime, track_loved,
album_loved, album_pop,
album_rate) = self.del_from_db(uri, False)
Logger.debug("CollectionScanner::add2db(): Read tags")
title = self.get_title(tags, name)
version = self.get_version(tags)
if version != "":
title += " (%s)" % version
artists = self.get_artists(tags)
a_sortnames = self.get_artist_sortnames(tags)
aa_sortnames = self.get_album_artist_sortnames(tags)
album_artists = self.get_album_artists(tags)
album_name = self.get_album_name(tags)
album_synced = 0
mb_album_id = self.get_mb_album_id(tags)
mb_track_id = self.get_mb_track_id(tags)
mb_artist_id = self.get_mb_artist_id(tags)
mb_album_artist_id = self.get_mb_album_artist_id(tags)
genres = self.get_genres(tags)
discnumber = self.get_discnumber(tags)
discname = self.get_discname(tags)
tracknumber = self.get_tracknumber(tags, name)
if track_rate == 0:
track_rate = self.get_popm(tags)
if album_mtime == 0:
album_mtime = track_mtime
bpm = self.get_bpm(tags)
compilation = self.get_compilation(tags)
year = None
if not App().settings.get_value("ignore-original-date"):
(year, timestamp) = self.get_original_year(tags)
if year is None:
(year, timestamp) = self.get_year(tags)
# If no artists tag, use album artist
if artists == "":
artists = album_artists
if App().settings.get_value("import-advanced-artist-tags"):
composers = self.get_composers(tags)
conductors = self.get_conductors(tags)
performers = self.get_performers(tags)
remixers = self.get_remixers(tags)
artists += ";%s" % performers if performers != "" else ""
artists += ";%s" % conductors if conductors != "" else ""
artists += ";%s" % composers if composers != "" else ""
artists += ";%s" % remixers if remixers != "" else ""
if artists == "":
artists = _("Unknown")
return (title, artists, genres, a_sortnames, aa_sortnames,
album_artists, album_name, discname, album_loved, album_mtime,
album_synced, album_rate, album_pop, discnumber, year,
timestamp, mb_album_id, mb_track_id, mb_artist_id,
mb_album_artist_id, tracknumber, track_pop, track_rate, bpm,
track_mtime, track_ltime, track_loved, duration, compilation)
def __add2db(self,
uri,
name,
artists,
genres,
a_sortnames,
aa_sortnames,
album_artists,
album_name,
discname,
album_loved,
album_mtime,
album_synced,
album_rate,
album_pop,
discnumber,
year,
timestamp,
mb_album_id,
mb_track_id,
mb_artist_id,
mb_album_artist_id,
tracknumber,
track_pop,
track_rate,
bpm,
track_mtime,
track_ltime,
track_loved,
duration,
compilation,
storage_type=StorageType.COLLECTION):
"""
Add new file to DB
@param uri as str
@param tags as *()
@param storage_type as StorageType
@return CollectionItem
"""
item = CollectionItem(uri=uri,
track_name=name,
artists=artists,
genres=genres,
a_sortnames=a_sortnames,
aa_sortnames=aa_sortnames,
album_artists=album_artists,
album_name=album_name,
discname=discname,
album_loved=album_loved,
album_mtime=album_mtime,
album_synced=album_synced,
album_rate=album_rate,
album_pop=album_pop,
discnumber=discnumber,
year=year,
timestamp=timestamp,
mb_album_id=mb_album_id,
mb_track_id=mb_track_id,
mb_artist_id=mb_artist_id,
mb_album_artist_id=mb_album_artist_id,
tracknumber=tracknumber,
track_pop=track_pop,
track_rate=track_rate,
bpm=bpm,
track_mtime=track_mtime,
track_ltime=track_ltime,
track_loved=track_loved,
duration=duration,
compilation=compilation,
storage_type=storage_type)
self.save_album(item)
self.save_track(item)
return item
