def clean_old_albums(self, storage_types):
"""
Clean old albums from DB
@param storage_types as [StorageType]
"""
SqlCursor.add(App().db)
# Remove older albums
for storage_type in storage_types:
# If too many albums, do some cleanup
count = App().albums.get_count_for_storage_type(storage_type)
diff = count - self.MAX_ITEMS_PER_STORAGE_TYPE
if diff > 0:
album_ids = App().albums.get_oldest_for_storage_type(
storage_type, diff)
for album_id in album_ids:
# EPHEMERAL with not tracks will be cleaned below
App().albums.set_storage_type(album_id,
StorageType.EPHEMERAL)
App().tracks.remove_album(album_id, False)
# On cancel, clean not needed, done in Application::quit()
if not self.__cancellable.is_cancelled():
App().tracks.clean(False)
App().albums.clean(False)
App().artists.clean(False)
SqlCursor.remove(App().db)
def __reset_database(self):
"""
Reset database
"""
def update_ui():
App().window.container.go_home()
App().scanner.update(ScanType.FULL)
App().player.stop()
if App().ws_director.collection_ws is not None:
App().ws_director.collection_ws.stop()
uris = App().tracks.get_uris()
i = 0
SqlCursor.add(App().db)
SqlCursor.add(self.__history)
count = len(uris)
for uri in uris:
self.del_from_db(uri, True)
self.__update_progress(i, count, 0.01)
i += 1
App().tracks.del_persistent(False)
App().tracks.clean(False)
App().albums.clean(False)
App().artists.clean(False)
App().genres.clean(False)
App().cache.clear_table("duration")
SqlCursor.commit(App().db)
SqlCursor.remove(App().db)
SqlCursor.commit(self.__history)
SqlCursor.remove(self.__history)
GLib.idle_add(update_ui)
def del_tracks(self, track_ids):
"""
Delete tracks from db
@param track_ids as [int]
"""
SqlCursor.add(Lp().playlists)
with SqlCursor(self) as sql:
all_album_ids = []
all_artist_ids = []
all_genre_ids = []
for track_id in track_ids:
album_id = Lp().tracks.get_album_id(track_id)
art_file = Lp().art.get_album_cache_name(Album(album_id))
genre_ids = Lp().tracks.get_genre_ids(track_id)
album_artist_ids = Lp().albums.get_artist_ids(album_id)
artist_ids = Lp().tracks.get_artist_ids(track_id)
uri = Lp().tracks.get_uri(track_id)
Lp().playlists.remove(uri)
Lp().tracks.remove(track_id)
Lp().tracks.clean(track_id)
all_album_ids.append(album_id)
all_artist_ids += album_artist_ids + artist_ids
all_genre_ids += genre_ids
for album_id in list(set(all_album_ids)):
if Lp().albums.clean(album_id):
Lp().art.clean_store(art_file)
for artist_id in list(set(all_artist_ids)):
Lp().artists.clean(artist_id)
for genre_id in list(set(all_genre_ids)):
Lp().genres.clean(genre_id)
sql.commit()
SqlCursor.remove(Lp().playlists)
def __vacuum(self):
"""
VACUUM DB
"""
try:
if self.scanner.is_locked():
self.scanner.stop()
GLib.idle_add(self.__vacuum)
return
SqlCursor.add(self.db)
self.tracks.del_non_persistent(False)
self.tracks.clean(False)
self.albums.clean(False)
self.artists.clean(False)
self.genres.clean(False)
SqlCursor.remove(self.db)
self.cache.clean(True)
with SqlCursor(self.db) as sql:
sql.isolation_level = None
sql.execute("VACUUM")
sql.isolation_level = ""
with SqlCursor(self.playlists) as sql:
sql.isolation_level = None
sql.execute("VACUUM")
sql.isolation_level = ""
except Exception as e:
Logger.error("Application::__vacuum(): %s" % e)
def upgrade(self, db):
"""
Upgrade db
@param db as Database
"""
version = 0
SqlCursor.add(db)
with SqlCursor(db, True) as sql:
result = sql.execute("PRAGMA user_version")
v = result.fetchone()
if v is not None:
version = v[0]
if version < self.version:
for i in range(version + 1, self.version + 1):
try:
if isinstance(self._UPGRADES[i], str):
sql.execute(self._UPGRADES[i])
SqlCursor.commit(db)
else:
self._UPGRADES[i](db)
SqlCursor.commit(db)
except Exception as e:
Logger.error("DB upgrade %s failed: %s" % (i, e))
sql.execute("PRAGMA user_version=%s" % self.version)
SqlCursor.remove(db)
def insert_track(self, playlist_id, track, position):
"""
Insert track at position, will remove track first if exists
@param playlist_id as int
@param track as Track
@param position as int
"""
SqlCursor.add(self)
track_ids = self.get_track_ids(playlist_id)
if track.id in track_ids:
index = track_ids.index(track.id)
track_ids.remove(track.id)
if index < position:
position -= 1
track_ids.insert(position, track.id)
self.clear(playlist_id)
tracks = [Track(track_id) for track_id in track_ids]
self.add_tracks(playlist_id, tracks)
SqlCursor.remove(self)
def __scan(self, uris, saved):
"""
Scan music collection for music files
@param uris as [str]
@param saved as bool
@thread safe
"""
modifications = False
if self.__history is None:
self.__history = History()
mtimes = App().tracks.get_mtimes()
(new_tracks, new_dirs) = self.__get_objects_for_uris(uris)
orig_tracks = App().tracks.get_uris()
was_empty = len(orig_tracks) == 0
count = len(new_tracks) + len(orig_tracks)
# Add monitors on dirs
if self.__inotify is not None:
for d in new_dirs:
if d.startswith("file://"):
self.__inotify.add_monitor(d)
i = 0
# Look for new files/modified file
SqlCursor.add(App().db)
try:
to_add = []
for uri in new_tracks:
if self.__thread is None:
SqlCursor.remove(App().db)
return
try:
GLib.idle_add(self.__update_progress, i, count)
f = Gio.File.new_for_uri(uri)
# We do not use time::modified because many tag editors
# just preserve this setting
try:
info = f.query_info("time::changed",
Gio.FileQueryInfoFlags.NONE, None)
mtime = int(
info.get_attribute_as_string("time::changed"))
except: # Fallback for remote fs
info = f.query_info("time::modified",
Gio.FileQueryInfoFlags.NONE, None)
mtime = int(
info.get_attribute_as_string("time::modified"))
# If songs exists and mtime unchanged, continue,
# else rescan
if uri in orig_tracks:
orig_tracks.remove(uri)
i += 1
if not saved or mtime <= mtimes.get(uri, mtime + 1):
i += 1
continue
else:
SqlCursor.allow_thread_execution(App().db)
self.__del_from_db(uri)
# If not saved, use 0 as mtime, easy delete on quit
if not saved:
mtime = 0
# On first scan, use modification time
# Else, use current time
elif not was_empty:
mtime = int(time())
to_add.append((uri, mtime))
except Exception as e:
Logger.error("CollectionScanner::__scan(mtime): %s" % e)
if to_add or orig_tracks:
modifications = True
# Clean deleted files
# Now because we need to populate history
# Only if we are saving
if saved:
for uri in orig_tracks:
i += 1
GLib.idle_add(self.__update_progress, i, count)
self.__del_from_db(uri)
SqlCursor.allow_thread_execution(App().db)
# Add files to db
for (uri, mtime) in to_add:
try:
Logger.debug("Adding file: %s" % uri)
i += 1
GLib.idle_add(self.__update_progress, i, count)
self.__add2db(uri, mtime)
SqlCursor.allow_thread_execution(App().db)
except Exception as e:
Logger.error("CollectionScanner::__scan(add): %s, %s" %
(e, uri))
except Exception as e:
Logger.error("CollectionScanner::__scan(): %s" % e)
SqlCursor.commit(App().db)
SqlCursor.remove(App().db)
GLib.idle_add(self.__finish, modifications and saved)
if not saved:
self.__play_new_tracks(new_tracks)
del self.__history
self.__history = None
def __scan(self, scan_type, uris):
"""
Scan music collection for music files
@param scan_type as ScanType
@param uris as [str]
@thread safe
"""
try:
SqlCursor.add(App().db)
App().art.clean_rounded()
(files, dirs,
streams) = self.__get_objects_for_uris(scan_type, uris)
if not files:
App().notify.send("Lollypop",
_("Scan disabled, missing collection"))
return
if scan_type == ScanType.NEW_FILES:
db_uris = App().tracks.get_uris(uris)
else:
db_uris = App().tracks.get_uris()
# Get mtime of all tracks to detect which has to be updated
db_mtimes = App().tracks.get_mtimes()
# * 2 => Scan + Save
self.__progress_total = len(files) * 2 + len(streams)
self.__progress_count = 0
self.__progress_fraction = 0
# Min: 1 thread, Max: 5 threads
count = max(1, min(5, cpu_count() // 2))
split_files = split_list(files, count)
self.__tags = {}
self.__pending_new_artist_ids = []
threads = []
for files in split_files:
thread = App().task_helper.run(self.__scan_files, files,
db_mtimes, scan_type)
threads.append(thread)
if scan_type == ScanType.EXTERNAL:
storage_type = StorageType.EXTERNAL
else:
storage_type = StorageType.COLLECTION
# Start getting files and populating DB
self.__items = []
i = 0
while threads:
thread = threads[i]
if not thread.is_alive():
threads.remove(thread)
self.__items += self.__save_in_db(storage_type)
if i >= len(threads) - 1:
i = 0
else:
i += 1
# Add streams to DB, only happening on command line/m3u files
self.__items += self.__save_streams_in_db(streams, storage_type)
self.__remove_old_tracks(db_uris, scan_type)
if scan_type == ScanType.EXTERNAL:
albums = tracks_to_albums(
[Track(item.track_id) for item in self.__items])
App().player.play_albums(albums)
else:
self.__add_monitor(dirs)
GLib.idle_add(self.__finish, self.__items)
self.__tags = {}
self.__items = []
self.__pending_new_artist_ids = []
except Exception as e:
Logger.warning("CollectionScanner::__scan(): %s", e)
SqlCursor.remove(App().db)
def __scan_files(self, files, db_uris, scan_type):
"""
Scan music collection for new audio files
@param files as [str]
@param db_uris as [str]
@param scan_type as ScanType
@return new track uris as [str]
@thread safe
"""
SqlCursor.add(App().db)
i = 0
# New tracks present in collection
new_tracks = []
# Get mtime of all tracks to detect which has to be updated
db_mtimes = App().tracks.get_mtimes()
count = len(files) + 1
try:
# Scan new files
for (mtime, uri) in files:
# Handle a stop request
if self.__thread is None and scan_type != ScanType.EPHEMERAL:
raise Exception("Scan add cancelled")
try:
if not self.__scan_to_handle(uri):
continue
if mtime > db_mtimes.get(uri, 0):
# If not saved, use 0 as mtime, easy delete on quit
if scan_type == ScanType.EPHEMERAL:
mtime = 0
# Do not use mtime if not intial scan
elif db_mtimes:
mtime = int(time())
Logger.debug("Adding file: %s" % uri)
self.__add2db(uri, mtime)
SqlCursor.allow_thread_execution(App().db)
new_tracks.append(uri)
except Exception as e:
Logger.error("CollectionScanner:: __scan_add_files: % s" %
e)
i += 1
self.__update_progress(i, count)
if scan_type != ScanType.EPHEMERAL and self.__thread is not None:
# We need to check files are always in collections
if scan_type == ScanType.FULL:
collections = App().settings.get_music_uris()
else:
collections = None
for uri in db_uris:
# Handle a stop request
if self.__thread is None:
raise Exception("Scan del cancelled")
in_collection = True
if collections is not None:
in_collection = False
for collection in collections:
if collection in uri:
in_collection = True
break
f = Gio.File.new_for_uri(uri)
if not in_collection or not f.query_exists():
self.del_from_db(uri, True)
SqlCursor.allow_thread_execution(App().db)
except Exception as e:
Logger.warning("CollectionScanner:: __scan_files: % s" % e)
SqlCursor.commit(App().db)
SqlCursor.remove(App().db)
return new_tracks
