class CollectionScanner(GObject.GObject, TagReader):
    """
        Scan user music collection
    """
    __gsignals__ = {
        "scan-finished": (GObject.SignalFlags.RUN_FIRST, None, (bool, )),
        "artist-updated": (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
        "genre-updated": (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
        "album-updated": (GObject.SignalFlags.RUN_FIRST, None, (int, bool))
    }

    def __init__(self):
        """
            Init collection scanner
        """
        GObject.GObject.__init__(self)
        TagReader.__init__(self)

        self.__thread = None
        self.__history = None
        self.__disable_compilations = True
        if App().settings.get_value("auto-update"):
            self.__inotify = Inotify()
        else:
            self.__inotify = None
        App().albums.update_max_count()

    def update(self, uris=[], saved=True):
        """
            Update database
            @param uris as [str]
            @param saved as bool
        """
        # Stop previous scan
        if self.is_locked():
            self.stop()
            GLib.timeout_add(250, self.update)
        else:
            self.__disable_compilations = not App().settings.get_value(
                "show-compilations")

            if not uris:
                uris = App().settings.get_music_uris()
                if not uris:
                    return

            App().window.container.progress.add(self)
            App().window.container.progress.set_fraction(0.0, self)

            self.__thread = Thread(target=self.__scan, args=(uris, saved))
            self.__thread.daemon = True
            self.__thread.start()

    def is_locked(self):
        """
            Return True if db locked
        """
        return self.__thread is not None and self.__thread.isAlive()

    def stop(self):
        """
            Stop scan
        """
        self.__thread = None

#######################
# PRIVATE             #
#######################

    def __get_objects_for_uris(self, uris):
        """
            Return all tracks/dirs for uris
            @param uris as string
            @return (track uri as [str], track dirs as [str])
        """
        tracks = []
        files = []
        track_dirs = []
        walk_uris = list(uris)
        while walk_uris:
            uri = walk_uris.pop(0)
            try:
                # Directly add files, walk through directories
                f = Gio.File.new_for_uri(uri)
                info = f.query_info(
                    "standard::name,standard::type,standard::is-hidden",
                    Gio.FileQueryInfoFlags.NONE, None)
                if info.get_file_type() == Gio.FileType.DIRECTORY:
                    track_dirs.append(uri)
                    infos = f.enumerate_children(
                        "standard::name,standard::type,standard::is-hidden",
                        Gio.FileQueryInfoFlags.NONE, None)
                    for info in infos:
                        f = infos.get_child(info)
                        child_uri = f.get_uri()
                        if info.get_is_hidden():
                            continue
                        elif info.get_file_type() == Gio.FileType.DIRECTORY:
                            track_dirs.append(child_uri)
                            walk_uris.append(child_uri)
                        else:
                            files.append(f)
                else:
                    files.append(f)
            except Exception as e:
                Logger.error(
                    "CollectionScanner::__get_objects_for_uris(): %s" % e)
        for f in files:
            try:
                if is_pls(f):
                    App().playlists.import_tracks(f)
                elif is_audio(f):
                    tracks.append(f.get_uri())
                else:
                    Logger.debug("%s not detected as a music file" %
                                 f.get_uri())
            except Exception as e:
                Logger.error(
                    "CollectionScanner::__get_objects_for_uris(): %s" % e)
        return (tracks, track_dirs)

    def __update_progress(self, current, total):
        """
            Update progress bar status
            @param scanned items as int, total items as int
        """
        App().window.container.progress.set_fraction(current / total, self)

    def __finish(self, modifications):
        """
            Notify from main thread when scan finished
            @param modifications as bool
        """
        App().window.container.progress.set_fraction(1.0, self)
        self.stop()
        self.emit("scan-finished", modifications)
        # Update max count value
        App().albums.update_max_count()
        if App().settings.get_value("artist-artwork"):
            App().art.cache_artists_info()

    def __scan(self, uris, saved):
        """
            Scan music collection for music files
            @param uris as [str]
            @param saved as bool
            @thread safe
        """
        modifications = False
        if self.__history is None:
            self.__history = History()
        mtimes = App().tracks.get_mtimes()
        (new_tracks, new_dirs) = self.__get_objects_for_uris(uris)
        orig_tracks = App().tracks.get_uris()
        was_empty = len(orig_tracks) == 0

        count = len(new_tracks) + len(orig_tracks)
        # Add monitors on dirs
        if self.__inotify is not None:
            for d in new_dirs:
                if d.startswith("file://"):
                    self.__inotify.add_monitor(d)

        i = 0
        # Look for new files/modified file
        SqlCursor.add(App().db)
        try:
            to_add = []
            for uri in new_tracks:
                if self.__thread is None:
                    SqlCursor.remove(App().db)
                    return
                try:
                    GLib.idle_add(self.__update_progress, i, count)
                    f = Gio.File.new_for_uri(uri)
                    # We do not use time::modified because many tag editors
                    # just preserve this setting
                    try:
                        info = f.query_info("time::changed",
                                            Gio.FileQueryInfoFlags.NONE, None)
                        mtime = int(
                            info.get_attribute_as_string("time::changed"))
                    except:  # Fallback for remote fs
                        info = f.query_info("time::modified",
                                            Gio.FileQueryInfoFlags.NONE, None)
                        mtime = int(
                            info.get_attribute_as_string("time::modified"))
                    # If songs exists and mtime unchanged, continue,
                    # else rescan
                    if uri in orig_tracks:
                        orig_tracks.remove(uri)
                        i += 1
                        if not saved or mtime <= mtimes.get(uri, mtime + 1):
                            i += 1
                            continue
                        else:
                            SqlCursor.allow_thread_execution(App().db)
                            self.__del_from_db(uri)
                    # If not saved, use 0 as mtime, easy delete on quit
                    if not saved:
                        mtime = 0
                    # On first scan, use modification time
                    # Else, use current time
                    elif not was_empty:
                        mtime = int(time())
                    to_add.append((uri, mtime))
                except Exception as e:
                    Logger.error("CollectionScanner::__scan(mtime): %s" % e)
            if to_add or orig_tracks:
                modifications = True
            # Clean deleted files
            # Now because we need to populate history
            # Only if we are saving
            if saved:
                for uri in orig_tracks:
                    i += 1
                    GLib.idle_add(self.__update_progress, i, count)
                    self.__del_from_db(uri)
                    SqlCursor.allow_thread_execution(App().db)
            # Add files to db
            for (uri, mtime) in to_add:
                try:
                    Logger.debug("Adding file: %s" % uri)
                    i += 1
                    GLib.idle_add(self.__update_progress, i, count)
                    self.__add2db(uri, mtime)
                    SqlCursor.allow_thread_execution(App().db)
                except Exception as e:
                    Logger.error("CollectionScanner::__scan(add): %s, %s" %
                                 (e, uri))
        except Exception as e:
            Logger.error("CollectionScanner::__scan(): %s" % e)
        SqlCursor.commit(App().db)
        SqlCursor.remove(App().db)
        GLib.idle_add(self.__finish, modifications and saved)
        if not saved:
            self.__play_new_tracks(new_tracks)
        del self.__history
        self.__history = None

    def __add2db(self, uri, mtime):
        """
            Add new file to db with information
            @param uri as string
            @param mtime as int
            @return track id as int
            @warning, be sure SqlCursor is available for App().db
        """
        f = Gio.File.new_for_uri(uri)
        Logger.debug("CollectionScanner::add2db(): Read tags")
        info = self.get_info(uri)
        tags = info.get_tags()
        name = f.get_basename()
        title = self.get_title(tags, name)
        version = self.get_version(tags)
        artists = self.get_artists(tags)
        composers = self.get_composers(tags)
        performers = self.get_performers(tags)
        a_sortnames = self.get_artist_sortnames(tags)
        aa_sortnames = self.get_album_artist_sortnames(tags)
        album_artists = self.get_album_artists(tags)
        album_name = self.get_album_name(tags)
        mb_album_id = self.get_mb_album_id(tags)
        mb_track_id = self.get_mb_track_id(tags)
        genres = self.get_genres(tags)
        discnumber = self.get_discnumber(tags)
        discname = self.get_discname(tags)
        tracknumber = self.get_tracknumber(tags, name)
        (year, timestamp) = self.get_original_year(tags)
        if year is None:
            (year, timestamp) = self.get_year(tags)
        duration = int(info.get_duration() / 1000000000)

        if version != "":
            title += " (%s)" % version

        # If no artists tag, use album artist
        if artists == "":
            artists = album_artists
        # if artists is always null, no album artists too,
        # use composer/performer
        if artists == "":
            artists = performers
            album_artists = composers
            if artists == "":
                artists = album_artists
            if artists == "":
                artists = _("Unknown")

        Logger.debug("CollectionScanner::add2db(): Restore stats")
        # Restore stats
        (track_pop, track_rate, track_ltime, album_mtime, track_loved,
         album_loved, album_pop,
         album_rate) = self.__history.get(name, duration)
        # If nothing in stats, use track mtime
        if album_mtime == 0:
            album_mtime = mtime

        Logger.debug("CollectionScanner::add2db(): Add artists %s" % artists)
        artist_ids = self.add_artists(artists, a_sortnames)

        Logger.debug("CollectionScanner::add2db(): "
                     "Add album artists %s" % album_artists)
        album_artist_ids = self.add_album_artists(album_artists, aa_sortnames)

        # User does not want compilations
        if self.__disable_compilations and not album_artist_ids:
            album_artist_ids = artist_ids

        missing_artist_ids = list(set(album_artist_ids) - set(artist_ids))
        # https://github.com/gnumdk/lollypop/issues/507#issuecomment-200526942
        # Special case for broken tags
        # Can't do more because don't want to break split album behaviour
        if len(missing_artist_ids) == len(album_artist_ids):
            artist_ids += missing_artist_ids

        Logger.debug("CollectionScanner::add2db(): Add album: "
                     "%s, %s" % (album_name, album_artist_ids))
        album_id = self.add_album(album_name, mb_album_id, album_artist_ids,
                                  uri, album_loved, album_pop, album_rate,
                                  mtime)

        genre_ids = self.add_genres(genres)

        # Add track to db
        Logger.debug("CollectionScanner::add2db(): Add track")
        track_id = App().tracks.add(title, uri, duration, tracknumber,
                                    discnumber, discname, album_id, year,
                                    timestamp, track_pop, track_rate,
                                    track_loved, track_ltime, mtime,
                                    mb_track_id)
        Logger.debug("CollectionScanner::add2db(): Update track")
        self.__update_track(track_id, artist_ids, genre_ids)
        Logger.debug("CollectionScanner::add2db(): Update album")
        SqlCursor.commit(App().db)
        self.__update_album(album_id, album_artist_ids, genre_ids, year,
                            timestamp)
        SqlCursor.commit(App().db)
        for genre_id in genre_ids:
            GLib.idle_add(self.emit, "genre-updated", genre_id, True)
        return track_id

    def __del_from_db(self, uri):
        """
            Delete track from db
            @param uri as str
        """
        try:
            f = Gio.File.new_for_uri(uri)
            name = f.get_basename()
            track_id = App().tracks.get_id_by_uri(uri)
            album_id = App().tracks.get_album_id(track_id)
            genre_ids = App().tracks.get_genre_ids(track_id)
            album_artist_ids = App().albums.get_artist_ids(album_id)
            artist_ids = App().tracks.get_artist_ids(track_id)
            popularity = App().tracks.get_popularity(track_id)
            rate = App().tracks.get_rate(track_id)
            ltime = App().tracks.get_ltime(track_id)
            mtime = App().tracks.get_mtime(track_id)
            loved_track = App().tracks.get_loved(track_id)
            duration = App().tracks.get_duration(track_id)
            album_popularity = App().albums.get_popularity(album_id)
            album_rate = App().albums.get_rate(album_id)
            loved_album = App().albums.get_loved(album_id)
            uri = App().tracks.get_uri(track_id)
            self.__history.add(name, duration, popularity, rate, ltime, mtime,
                               loved_track, loved_album, album_popularity,
                               album_rate)
            App().tracks.remove(track_id)
            App().tracks.clean(track_id)
            cleaned = App().albums.clean(album_id)
            if cleaned:
                SqlCursor.commit(App().db)
                GLib.idle_add(self.emit, "album-updated", album_id, True)
            for artist_id in album_artist_ids + artist_ids:
                cleaned = App().artists.clean(artist_id)
                # Force update even if not cleaned as artist may
                # have been removed from a selected genre
                GLib.idle_add(self.emit, "artist-updated", artist_id, False)
            for genre_id in genre_ids:
                cleaned = App().genres.clean(genre_id)
                if cleaned:
                    SqlCursor.commit(App().db)
                    GLib.idle_add(self.emit, "genre-updated", genre_id, False)
        except Exception as e:
            Logger.error("CollectionScanner::__del_from_db: %s" % e)

    def __update_album(self, album_id, artist_ids, genre_ids, year, timestamp):
        """
            Update album artists based on album-artist and artist tags
            This code auto handle compilations: empty "album artist" with
            different artists
            @param album id as int
            @param artist ids as [int]
            @param genre ids as [int]
            @param year as int
            @param timestmap as int
            @commit needed
        """
        album_artist_ids = []
        add = True
        # Set artist ids based on content
        if not artist_ids:
            new_artist_ids = App().albums.calculate_artist_ids(album_id)
            current_artist_ids = App().albums.get_artist_ids(album_id)
            if new_artist_ids != current_artist_ids:
                album_artist_ids = new_artist_ids
                if Type.COMPILATIONS in new_artist_ids:
                    add = False
                    album_artist_ids = current_artist_ids
                else:
                    album_artist_ids = new_artist_ids
                App().albums.set_artist_ids(album_id, new_artist_ids)
        # Update UI based on previous artist calculation
        for artist_id in album_artist_ids:
            GLib.idle_add(self.emit, "artist-updated", artist_id, add)
        # Update album genres
        for genre_id in genre_ids:
            App().albums.add_genre(album_id, genre_id)

        # Update year based on tracks
        year = App().tracks.get_year_for_album(album_id)
        App().albums.set_year(album_id, year)
        timestamp = App().tracks.get_timestamp_for_album(album_id)
        App().albums.set_timestamp(album_id, timestamp)

    def __update_track(self, track_id, artist_ids, genre_ids):
        """
            Set track artists/genres
            @param track id as int
            @param artist ids as [int]
            @param genre ids as [int]
            @param mtime as int
            @param popularity as int
            @commit needed
        """
        # Set artists/genres for track
        for artist_id in artist_ids:
            App().tracks.add_artist(track_id, artist_id)
        for genre_id in genre_ids:
            App().tracks.add_genre(track_id, genre_id)

    def __play_new_tracks(self, uris):
        """
            Play new tracks
            @param uri as [str]
        """
        # First get tracks
        tracks = []
        for uri in uris:
            track_id = App().tracks.get_id_by_uri(uri)
            tracks.append(Track(track_id))
        # Then get album ids
        album_ids = {}
        for track in tracks:
            if track.album.id in album_ids.keys():
                album_ids[track.album.id].append(track)
            else:
                album_ids[track.album.id] = [track]
        # Create albums with tracks
        play = True
        for album_id in album_ids.keys():
            album = Album(album_id)
            album.set_tracks(album_ids[album_id])
            if play:
                App().player.play_album(album)
            else:
                App().player.add_album(album)
class CollectionScanner(GObject.GObject, TagReader):
    """
        Scan user music collection
    """
    __gsignals__ = {
        "scan-finished": (GObject.SignalFlags.RUN_FIRST, None, (bool, )),
        "updated":
        (GObject.SignalFlags.RUN_FIRST, None, (GObject.TYPE_PYOBJECT, int))
    }

    def __init__(self):
        """
            Init collection scanner
        """
        GObject.GObject.__init__(self)
        self.__thread = None
        self.__tags = {}
        self.__items = []
        self.__pending_new_artist_ids = []
        self.__history = History()
        self.__progress_total = 1
        self.__progress_count = 0
        self.__progress_fraction = 0
        self.__disable_compilations = not App().settings.get_value(
            "show-compilations")
        if App().settings.get_value("auto-update"):
            self.__inotify = Inotify()
        else:
            self.__inotify = None
        App().albums.update_max_count()

    def update(self, scan_type, uris=[]):
        """
            Update database
            @param scan_type as ScanType
            @param uris as [str]
        """
        self.__disable_compilations = not App().settings.get_value(
            "show-compilations")
        App().lookup_action("update_db").set_enabled(False)
        # Stop previous scan
        if self.is_locked() and scan_type != ScanType.EXTERNAL:
            self.stop()
            GLib.timeout_add(250, self.update, scan_type, uris)
        elif App().ws_director.collection_ws is not None and\
                not App().ws_director.collection_ws.stop():
            GLib.timeout_add(250, self.update, scan_type, uris)
        else:
            if scan_type == ScanType.FULL:
                uris = App().settings.get_music_uris()
            if not uris:
                return
            # Register to progressbar
            if scan_type != ScanType.EXTERNAL:
                App().window.container.progress.add(self)
                App().window.container.progress.set_fraction(0, self)
            Logger.info("Scan started")
            # Launch scan in a separate thread
            self.__thread = App().task_helper.run(self.__scan, scan_type, uris)

    def save_album(
        self,
        item,
    ):
        """
            Add album to DB
            @param item as CollectionItem
        """
        Logger.debug("CollectionScanner::save_album(): "
                     "Add album artists %s" % item.album_artists)
        (item.new_album_artist_ids,
         item.album_artist_ids) = self.add_artists(item.album_artists,
                                                   item.aa_sortnames,
                                                   item.mb_album_artist_id)
        # We handle artists already created by any previous save_track()
        for artist_id in item.album_artist_ids:
            if artist_id in self.__pending_new_artist_ids:
                item.new_album_artist_ids.append(artist_id)
                self.__pending_new_artist_ids.remove(artist_id)

        item.lp_album_id = get_lollypop_album_id(item.album_name,
                                                 item.album_artists)
        Logger.debug("CollectionScanner::save_track(): Add album: "
                     "%s, %s" % (item.album_name, item.album_artist_ids))
        (item.new_album, item.album_id) = self.add_album(
            item.album_name, item.mb_album_id, item.lp_album_id,
            item.album_artist_ids, item.uri, item.album_loved, item.album_pop,
            item.album_rate, item.album_synced, item.album_mtime,
            item.storage_type)
        if item.year is not None:
            App().albums.set_year(item.album_id, item.year)
            App().albums.set_timestamp(item.album_id, item.timestamp)

    def save_track(self, item):
        """
            Add track to DB
            @param item as CollectionItem
        """
        Logger.debug("CollectionScanner::save_track(): Add artists %s" %
                     item.artists)
        (item.new_artist_ids,
         item.artist_ids) = self.add_artists(item.artists, item.a_sortnames,
                                             item.mb_artist_id)

        self.__pending_new_artist_ids += item.new_artist_ids
        missing_artist_ids = list(
            set(item.album_artist_ids) - set(item.artist_ids))
        # Special case for broken tags
        # If all artist album tags are missing
        # Can't do more because don't want to break split album behaviour
        if len(missing_artist_ids) == len(item.album_artist_ids):
            item.artist_ids += missing_artist_ids

        if item.genres is None:
            (item.new_genre_ids, item.genre_ids) = ([], [Type.WEB])
        else:
            (item.new_genre_ids, item.genre_ids) = self.add_genres(item.genres)

        item.lp_track_id = get_lollypop_track_id(item.track_name, item.artists,
                                                 item.album_name)

        # Add track to db
        Logger.debug("CollectionScanner::save_track(): Add track")
        item.track_id = App().tracks.add(
            item.track_name, item.uri, item.duration, item.tracknumber,
            item.discnumber, item.discname, item.album_id, item.year,
            item.timestamp, item.track_pop, item.track_rate, item.track_loved,
            item.track_ltime, item.track_mtime, item.mb_track_id,
            item.lp_track_id, item.bpm, item.storage_type)
        Logger.debug("CollectionScanner::save_track(): Update track")
        self.update_track(item)
        Logger.debug("CollectionScanner::save_track(): Update album")
        self.update_album(item)

    def update_album(self, item):
        """
            Update album artists based on album-artist and artist tags
            This code auto handle compilations: empty "album artist" with
            different artists
            @param item as CollectionItem
        """
        if item.album_artist_ids:
            App().albums.set_artist_ids(item.album_id, item.album_artist_ids)
        # Set artist ids based on content
        else:
            new_album_artist_ids = App().albums.calculate_artist_ids(
                item.album_id, self.__disable_compilations)
            App().albums.set_artist_ids(item.album_id, new_album_artist_ids)
            # We handle artists already created by any previous save_track()
            item.new_album_artist_ids = []
            for artist_id in new_album_artist_ids:
                if artist_id in self.__pending_new_artist_ids:
                    item.new_album_artist_ids.append(artist_id)
                    self.__pending_new_artist_ids.remove(artist_id)
        # Update lp_album_id
        lp_album_id = get_lollypop_album_id(item.album_name,
                                            item.album_artists)
        if lp_album_id != item.lp_album_id:
            App().art.move_artwork(item.lp_album_id, lp_album_id)
            App().albums.set_lp_album_id(item.album_id, lp_album_id)
            item.lp_album_id = lp_album_id
        # Update album genres
        for genre_id in item.genre_ids:
            App().albums.add_genre(item.album_id, genre_id)
        # Update year based on tracks
        year = App().tracks.get_year_for_album(item.album_id)
        if year is not None:
            App().albums.set_year(item.album_id, year)
            timestamp = App().tracks.get_timestamp_for_album(item.album_id)
            App().albums.set_timestamp(item.album_id, timestamp)
        App().cache.clear_durations(item.album_id)

    def update_track(self, item):
        """
            Set track artists/genres
            @param item as CollectionItem
        """
        # Set artists/genres for track
        for artist_id in item.artist_ids:
            App().tracks.add_artist(item.track_id, artist_id)
        for genre_id in item.genre_ids:
            App().tracks.add_genre(item.track_id, genre_id)

    def del_from_db(self, uri, backup):
        """
            Delete track from db
            @param uri as str
            @param backup as bool
            @return (popularity, ltime, mtime,
                     loved album, album_popularity)
        """
        try:
            track_id = App().tracks.get_id_by_uri(uri)
            duration = App().tracks.get_duration(track_id)
            album_id = App().tracks.get_album_id(track_id)
            album_artist_ids = App().albums.get_artist_ids(album_id)
            artist_ids = App().tracks.get_artist_ids(track_id)
            track_pop = App().tracks.get_popularity(track_id)
            track_rate = App().tracks.get_rate(track_id)
            track_ltime = App().tracks.get_ltime(track_id)
            album_mtime = App().tracks.get_mtime(track_id)
            track_loved = App().tracks.get_loved(track_id)
            album_pop = App().albums.get_popularity(album_id)
            album_rate = App().albums.get_rate(album_id)
            album_loved = App().albums.get_loved(album_id)
            album_synced = App().albums.get_synced(album_id)
            if backup:
                f = Gio.File.new_for_uri(uri)
                name = f.get_basename()
                self.__history.add(name, duration, track_pop, track_rate,
                                   track_ltime, album_mtime, track_loved,
                                   album_loved, album_pop, album_rate,
                                   album_synced)
            App().tracks.remove(track_id)
            genre_ids = App().tracks.get_genre_ids(track_id)
            App().albums.clean()
            App().genres.clean()
            App().artists.clean()
            App().cache.clear_durations(album_id)
            SqlCursor.commit(App().db)
            item = CollectionItem(album_id=album_id)
            if not App().albums.get_name(album_id):
                item.artist_ids = []
                for artist_id in album_artist_ids + artist_ids:
                    if not App().artists.get_name(artist_id):
                        item.artist_ids.append(artist_id)
                item.genre_ids = []
                for genre_id in genre_ids:
                    if not App().genres.get_name(genre_id):
                        item.genre_ids.append(genre_id)
                emit_signal(self, "updated", item, ScanUpdate.REMOVED)
            else:
                # Force genre for album
                genre_ids = App().tracks.get_album_genre_ids(album_id)
                App().albums.set_genre_ids(album_id, genre_ids)
                emit_signal(self, "updated", item, ScanUpdate.MODIFIED)
            return (track_pop, track_rate, track_ltime, album_mtime,
                    track_loved, album_loved, album_pop, album_rate)
        except Exception as e:
            Logger.error("CollectionScanner::del_from_db: %s" % e)

    def is_locked(self):
        """
            True if db locked
            @return bool
        """
        return self.__thread is not None and self.__thread.is_alive()

    def stop(self):
        """
            Stop scan
        """
        self.__thread = None

    def reset_database(self):
        """
            Reset database
        """
        from lollypop.app_notification import AppNotification
        App().window.container.progress.add(self)
        App().window.container.progress.set_fraction(0, self)
        self.__progress_fraction = 0
        notification = AppNotification(_("Resetting database"), [], [])
        notification.show()
        App().window.container.add_overlay(notification)
        notification.set_reveal_child(True)
        App().task_helper.run(self.__reset_database)

    @property
    def inotify(self):
        """
            Get Inotify object
            @return Inotify
        """
        return self.__inotify

#######################
# PRIVATE             #
#######################

    def __reset_database(self):
        """
            Reset database
        """
        def update_ui():
            App().window.container.go_home()
            App().scanner.update(ScanType.FULL)

        App().player.stop()
        if App().ws_director.collection_ws is not None:
            App().ws_director.collection_ws.stop()
        uris = App().tracks.get_uris()
        i = 0
        SqlCursor.add(App().db)
        SqlCursor.add(self.__history)
        count = len(uris)
        for uri in uris:
            self.del_from_db(uri, True)
            self.__update_progress(i, count, 0.01)
            i += 1
        App().tracks.del_persistent(False)
        App().tracks.clean(False)
        App().albums.clean(False)
        App().artists.clean(False)
        App().genres.clean(False)
        App().cache.clear_table("duration")
        SqlCursor.commit(App().db)
        SqlCursor.remove(App().db)
        SqlCursor.commit(self.__history)
        SqlCursor.remove(self.__history)
        GLib.idle_add(update_ui)

    def __update_progress(self, current, total, allowed_diff):
        """
            Update progress bar status
            @param current as int
            @param total as int
            @param allowed_diff as float => allows to prevent
                                            main loop flooding
        """
        new_fraction = current / total
        if new_fraction > self.__progress_fraction + allowed_diff:
            self.__progress_fraction = new_fraction
            GLib.idle_add(App().window.container.progress.set_fraction,
                          new_fraction, self)

    def __finish(self, items):
        """
            Notify from main thread when scan finished
            @param items as [CollectionItem]
        """
        track_ids = [item.track_id for item in items]
        self.__thread = None
        Logger.info("Scan finished")
        App().lookup_action("update_db").set_enabled(True)
        App().window.container.progress.set_fraction(1.0, self)
        self.stop()
        emit_signal(self, "scan-finished", track_ids)
        # Update max count value
        App().albums.update_max_count()
        # Update featuring
        App().artists.update_featuring()
        if App().ws_director.collection_ws is not None:
            App().ws_director.collection_ws.start()

    def __add_monitor(self, dirs):
        """
            Monitor any change in a list of directory
            @param dirs as str or list of directory to be monitored
        """
        if self.__inotify is None:
            return
        # Add monitors on dirs
        for d in dirs:
            # Handle a stop request
            if self.__thread is None:
                break
            if d.startswith("file://"):
                self.__inotify.add_monitor(d)

    @profile
    def __get_objects_for_uris(self, scan_type, uris):
        """
            Get all tracks and dirs in uris
            @param scan_type as ScanType
            @param uris as string
            @return ([(int, str)], [str], [str])
                    ([(mtime, file)], [dir], [stream])
        """
        files = []
        dirs = []
        streams = []
        walk_uris = []
        # Check collection exists
        for uri in uris:
            parsed = urlparse(uri)
            if parsed.scheme in ["http", "https"]:
                streams.append(uri)
            else:
                f = Gio.File.new_for_uri(uri)
                if f.query_exists():
                    walk_uris.append(uri)
                else:
                    return ([], [], [])

        while walk_uris:
            uri = walk_uris.pop(0)
            try:
                # Directly add files, walk through directories
                f = Gio.File.new_for_uri(uri)
                info = f.query_info(SCAN_QUERY_INFO,
                                    Gio.FileQueryInfoFlags.NONE, None)
                if info.get_file_type() == Gio.FileType.DIRECTORY:
                    dirs.append(uri)
                    infos = f.enumerate_children(SCAN_QUERY_INFO,
                                                 Gio.FileQueryInfoFlags.NONE,
                                                 None)
                    for info in infos:
                        f = infos.get_child(info)
                        child_uri = f.get_uri()
                        if info.get_is_hidden():
                            continue
                        # User do not want internal symlinks
                        elif info.get_is_symlink() and\
                                App().settings.get_value("ignore-symlinks"):
                            continue
                        walk_uris.append(child_uri)
                    infos.close(None)
                # Only happens if files passed as args
                else:
                    mtime = get_mtime(info)
                    files.append((mtime, uri))
            except Exception as e:
                Logger.error(
                    "CollectionScanner::__get_objects_for_uris(): %s" % e)
        files.sort(reverse=True)
        return (files, dirs, streams)

    @profile
    def __scan(self, scan_type, uris):
        """
            Scan music collection for music files
            @param scan_type as ScanType
            @param uris as [str]
            @thread safe
        """
        try:
            SqlCursor.add(App().db)
            App().art.clean_rounded()
            (files, dirs,
             streams) = self.__get_objects_for_uris(scan_type, uris)
            if not files:
                App().notify.send("Lollypop",
                                  _("Scan disabled, missing collection"))
                return
            if scan_type == ScanType.NEW_FILES:
                db_uris = App().tracks.get_uris(uris)
            else:
                db_uris = App().tracks.get_uris()

            # Get mtime of all tracks to detect which has to be updated
            db_mtimes = App().tracks.get_mtimes()
            # * 2 => Scan + Save
            self.__progress_total = len(files) * 2 + len(streams)
            self.__progress_count = 0
            self.__progress_fraction = 0
            # Min: 1 thread, Max: 5 threads
            count = max(1, min(5, cpu_count() // 2))
            split_files = split_list(files, count)
            self.__tags = {}
            self.__pending_new_artist_ids = []
            threads = []
            for files in split_files:
                thread = App().task_helper.run(self.__scan_files, files,
                                               db_mtimes, scan_type)
                threads.append(thread)

            if scan_type == ScanType.EXTERNAL:
                storage_type = StorageType.EXTERNAL
            else:
                storage_type = StorageType.COLLECTION
            # Start getting files and populating DB
            self.__items = []
            i = 0
            while threads:
                thread = threads[i]
                if not thread.is_alive():
                    threads.remove(thread)
                self.__items += self.__save_in_db(storage_type)
                if i >= len(threads) - 1:
                    i = 0
                else:
                    i += 1

            # Add streams to DB, only happening on command line/m3u files
            self.__items += self.__save_streams_in_db(streams, storage_type)

            self.__remove_old_tracks(db_uris, scan_type)

            if scan_type == ScanType.EXTERNAL:
                albums = tracks_to_albums(
                    [Track(item.track_id) for item in self.__items])
                App().player.play_albums(albums)
            else:
                self.__add_monitor(dirs)
                GLib.idle_add(self.__finish, self.__items)
            self.__tags = {}
            self.__items = []
            self.__pending_new_artist_ids = []
        except Exception as e:
            Logger.warning("CollectionScanner::__scan(): %s", e)
        SqlCursor.remove(App().db)

    def __scan_to_handle(self, uri):
        """
            Check if file has to be handle by scanner
            @param f as Gio.File
            @return bool
        """
        try:
            file_type = get_file_type(uri)
            # Get file type using Gio (slower)
            if file_type == FileType.UNKNOWN:
                f = Gio.File.new_for_uri(uri)
                info = f.query_info(FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE,
                                    Gio.FileQueryInfoFlags.NONE)
                if is_pls(info):
                    file_type = FileType.PLS
                elif is_audio(info):
                    file_type = FileType.AUDIO
            if file_type == FileType.PLS:
                Logger.debug("Importing playlist %s" % uri)
                if App().settings.get_value("import-playlists"):
                    App().playlists.import_tracks(uri)
            elif file_type == FileType.AUDIO:
                Logger.debug("Importing audio %s" % uri)
                return True
        except Exception as e:
            Logger.error("CollectionScanner::__scan_to_handle(): %s" % e)
        return False

    def __scan_files(self, files, db_mtimes, scan_type):
        """
            Scan music collection for new audio files
            @param files as [str]
            @param db_mtimes as {}
            @param scan_type as ScanType
            @thread safe
        """
        discoverer = Discoverer()
        try:
            # Scan new files
            for (mtime, uri) in files:
                # Handle a stop request
                if self.__thread is None and scan_type != ScanType.EXTERNAL:
                    raise Exception("cancelled")
                try:
                    if not self.__scan_to_handle(uri):
                        self.__progress_count += 2
                        continue
                    db_mtime = db_mtimes.get(uri, 0)
                    if mtime > db_mtime:
                        # Do not use mtime if not intial scan
                        if db_mtimes:
                            mtime = int(time())
                        self.__tags[uri] = self.__get_tags(
                            discoverer, uri, mtime)
                        self.__progress_count += 1
                        self.__update_progress(self.__progress_count,
                                               self.__progress_total, 0.001)
                    else:
                        # We want to play files, so put them in items
                        if scan_type == ScanType.EXTERNAL:
                            track_id = App().tracks.get_id_by_uri(uri)
                            item = CollectionItem(track_id=track_id)
                            self.__items.append(item)
                        self.__progress_count += 2
                        self.__update_progress(self.__progress_count,
                                               self.__progress_total, 0.1)
                except Exception as e:
                    Logger.error("Scanning file: %s, %s" % (uri, e))
        except Exception as e:
            Logger.warning("CollectionScanner::__scan_files(): % s" % e)

    def __save_in_db(self, storage_type):
        """
            Save current tags into DB
            @param storage_type as StorageType
            @return [CollectionItem]
        """
        items = []
        notify_index = 0
        previous_album_id = None
        for uri in list(self.__tags.keys()):
            # Handle a stop request
            if self.__thread is None:
                raise Exception("cancelled")
            Logger.debug("Adding file: %s" % uri)
            tags = self.__tags[uri]
            item = self.__add2db(uri, *tags, storage_type)
            items.append(item)
            self.__progress_count += 1
            self.__update_progress(self.__progress_count,
                                   self.__progress_total, 0.001)
            if previous_album_id != item.album_id:
                self.__notify_ui(items[notify_index:])
                notify_index = len(items)
                previous_album_id = item.album_id
            del self.__tags[uri]
        # Handle a stop request
        if self.__thread is None:
            raise Exception("cancelled")
        self.__notify_ui(items)
        return items

    def __save_streams_in_db(self, streams, storage_type):
        """
            Save http stream to DB
            @param streams as [str]
            @param storage_type as StorageType
            @return [CollectionItem]
        """
        items = []
        for uri in streams:
            parsed = urlparse(uri)
            item = self.__add2db(uri, parsed.path, parsed.netloc, None, "", "",
                                 parsed.netloc, parsed.netloc, "", False, 0,
                                 False, 0, 0, 0, None, 0, "", "", "", "", 1, 0,
                                 0, 0, 0, 0, False, 0, storage_type)
            items.append(item)
            self.__progress_count += 1
        return items

    def __notify_ui(self, items):
        """
            Notify UI based on current items
            @param items as [CollectionItem]
        """
        SqlCursor.commit(App().db)
        for item in items:
            if item.new_album:
                emit_signal(self, "updated", item, ScanUpdate.ADDED)
            else:
                emit_signal(self, "updated", item, ScanUpdate.MODIFIED)

    def __remove_old_tracks(self, uris, scan_type):
        """
            Remove non existent tracks from DB
            @param scan_type as ScanType
        """
        if scan_type != ScanType.EXTERNAL and self.__thread is not None:
            # We need to check files are always in collections
            if scan_type == ScanType.FULL:
                collections = App().settings.get_music_uris()
            else:
                collections = None
            for uri in uris:
                # Handle a stop request
                if self.__thread is None:
                    raise Exception("cancelled")
                in_collection = True
                if collections is not None:
                    in_collection = False
                    for collection in collections:
                        if collection in uri:
                            in_collection = True
                            break
                f = Gio.File.new_for_uri(uri)
                if not in_collection:
                    Logger.warning(
                        "Removed, not in collection anymore: %s -> %s", uri,
                        collections)
                    self.del_from_db(uri, True)
                elif not f.query_exists():
                    Logger.warning("Removed, file has been deleted: %s", uri)
                    self.del_from_db(uri, True)

    def __get_tags(self, discoverer, uri, track_mtime):
        """
            Read track tags
            @param discoverer as Discoverer
            @param uri as string
            @param track_mtime as int
            @return ()
        """
        f = Gio.File.new_for_uri(uri)
        info = discoverer.get_info(uri)
        tags = info.get_tags()
        name = f.get_basename()
        duration = int(info.get_duration() / 1000000)
        Logger.debug("CollectionScanner::add2db(): Restore stats")
        # Restore stats
        track_id = App().tracks.get_id_by_uri(uri)
        if track_id is None:
            track_id = App().tracks.get_id_by_basename_duration(name, duration)
        if track_id is None:
            (track_pop, track_rate, track_ltime, album_mtime, track_loved,
             album_loved, album_pop, album_rate,
             album_synced) = self.__history.get(name, duration)
        # Delete track and restore from it
        else:
            (track_pop, track_rate, track_ltime, album_mtime, track_loved,
             album_loved, album_pop,
             album_rate) = self.del_from_db(uri, False)

        Logger.debug("CollectionScanner::add2db(): Read tags")
        title = self.get_title(tags, name)
        version = self.get_version(tags)
        if version != "":
            title += " (%s)" % version
        artists = self.get_artists(tags)
        composers = self.get_composers(tags)
        performers = self.get_performers(tags)
        remixers = self.get_remixers(tags)
        if remixers != "":
            artists += ";%s" % remixers
        a_sortnames = self.get_artist_sortnames(tags)
        aa_sortnames = self.get_album_artist_sortnames(tags)
        album_artists = self.get_album_artists(tags)
        album_name = self.get_album_name(tags)
        album_synced = 0
        mb_album_id = self.get_mb_album_id(tags)
        mb_track_id = self.get_mb_track_id(tags)
        mb_artist_id = self.get_mb_artist_id(tags)
        mb_album_artist_id = self.get_mb_album_artist_id(tags)
        genres = self.get_genres(tags)
        discnumber = self.get_discnumber(tags)
        discname = self.get_discname(tags)
        tracknumber = self.get_tracknumber(tags, name)
        if track_rate == 0:
            track_rate = self.get_popm(tags)
        if album_mtime == 0:
            album_mtime = track_mtime
        bpm = self.get_bpm(tags)
        year = None
        if not App().settings.get_value("ignore-original-date"):
            (year, timestamp) = self.get_original_year(tags)
        if year is None:
            (year, timestamp) = self.get_year(tags)
        # If no artists tag, use album artist
        if artists == "":
            artists = album_artists
        # if artists is always null, no album artists too,
        # use composer/performer
        if artists == "":
            artists = performers
            album_artists = composers
            if artists == "":
                artists = album_artists
            if artists == "":
                artists = _("Unknown")
        return (title, artists, genres, a_sortnames, aa_sortnames,
                album_artists, album_name, discname, album_loved, album_mtime,
                album_synced, album_rate, album_pop, discnumber, year,
                timestamp, mb_album_id, mb_track_id, mb_artist_id,
                mb_album_artist_id, tracknumber, track_pop, track_rate, bpm,
                track_mtime, track_ltime, track_loved, duration)

    def __add2db(self,
                 uri,
                 name,
                 artists,
                 genres,
                 a_sortnames,
                 aa_sortnames,
                 album_artists,
                 album_name,
                 discname,
                 album_loved,
                 album_mtime,
                 album_synced,
                 album_rate,
                 album_pop,
                 discnumber,
                 year,
                 timestamp,
                 mb_album_id,
                 mb_track_id,
                 mb_artist_id,
                 mb_album_artist_id,
                 tracknumber,
                 track_pop,
                 track_rate,
                 bpm,
                 track_mtime,
                 track_ltime,
                 track_loved,
                 duration,
                 storage_type=StorageType.COLLECTION):
        """
            Add new file to DB
            @param uri as str
            @param tags as *()
            @param storage_type as StorageType
            @return CollectionItem
        """
        item = CollectionItem(uri=uri,
                              track_name=name,
                              artists=artists,
                              genres=genres,
                              a_sortnames=a_sortnames,
                              aa_sortnames=aa_sortnames,
                              album_artists=album_artists,
                              album_name=album_name,
                              discname=discname,
                              album_loved=album_loved,
                              album_mtime=album_mtime,
                              album_synced=album_synced,
                              album_rate=album_rate,
                              album_pop=album_pop,
                              discnumber=discnumber,
                              year=year,
                              timestamp=timestamp,
                              mb_album_id=mb_album_id,
                              mb_track_id=mb_track_id,
                              mb_artist_id=mb_artist_id,
                              mb_album_artist_id=mb_album_artist_id,
                              tracknumber=tracknumber,
                              track_pop=track_pop,
                              track_rate=track_rate,
                              bpm=bpm,
                              track_mtime=track_mtime,
                              track_ltime=track_ltime,
                              track_loved=track_loved,
                              duration=duration,
                              storage_type=storage_type)
        self.save_album(item)
        self.save_track(item)
        return item
Ejemplo n.º 3
0
class CollectionScanner(GObject.GObject, TagReader):
    """
        Scan user music collection
    """
    __gsignals__ = {
        'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
        'artist-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
        'genre-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
        'album-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool))
    }

    def __init__(self):
        """
            Init collection scanner
        """
        GObject.GObject.__init__(self)
        TagReader.__init__(self)

        self.__thread = None
        self.__history = None
        if Lp().settings.get_value('auto-update'):
            self.__inotify = Inotify()
        else:
            self.__inotify = None

    def update(self):
        """
            Update database
        """
        if not self.is_locked():
            uris = Lp().settings.get_music_uris()
            if not uris:
                return

            Lp().window.progress.add(self)
            Lp().window.progress.set_fraction(0.0, self)

            if Lp().notify is not None:
                Lp().notify.send(_("Your music is updating"))
            self.__thread = Thread(target=self.__scan, args=(uris,))
            self.__thread.daemon = True
            self.__thread.start()

    def clean_charts(self):
        """
            Clean charts in db
        """
        self.__thread = Thread(target=self.__clean_charts)
        self.__thread.daemon = True
        self.__thread.start()

    def is_locked(self):
        """
            Return True if db locked
        """
        return self.__thread is not None and self.__thread.isAlive()

    def stop(self):
        """
            Stop scan
        """
        self.__thread = None

#######################
# PRIVATE             #
#######################
    def __clean_charts(self):
        """
            Clean charts in db
        """
        track_ids = Lp().tracks.get_charts()
        Lp().db.del_tracks(track_ids)
        self.stop()

    def __get_objects_for_uris(self, uris):
        """
            Return all tracks/dirs for uris
            @param uris as string
            @return (track uri as [str], track dirs as [str],
                     ignore dirs as [str])
        """
        tracks = []
        ignore_dirs = []
        track_dirs = list(uris)
        walk_uris = list(uris)
        while walk_uris:
            uri = walk_uris.pop(0)
            empty = True
            try:
                d = Gio.File.new_for_uri(uri)
                infos = d.enumerate_children(
                    'standard::name,standard::type',
                    Gio.FileQueryInfoFlags.NONE,
                    None)
            except Exception as e:
                print("CollectionScanner::__get_objects_for_uris():", e)
                continue
            for info in infos:
                f = infos.get_child(info)
                child_uri = f.get_uri()
                empty = False
                if info.get_file_type() == Gio.FileType.DIRECTORY:
                    track_dirs.append(child_uri)
                    walk_uris.append(child_uri)
                else:
                    try:
                        f = Gio.File.new_for_uri(child_uri)
                        if is_pls(f):
                            pass
                        elif is_audio(f):
                            tracks.append(child_uri)
                        else:
                            debug("%s not detected as a music file" % uri)
                    except Exception as e:
                        print("CollectionScanner::"
                              "__get_objects_for_uris():", e)
            # If a root uri is empty
            # Ensure user is not doing something bad
            if empty and uri in uris:
                ignore_dirs.append(uri)
        return (tracks, track_dirs, ignore_dirs)

    def __update_progress(self, current, total):
        """
            Update progress bar status
            @param scanned items as int, total items as int
        """
        Lp().window.progress.set_fraction(current / total, self)

    def __finish(self):
        """
            Notify from main thread when scan finished
        """
        Lp().window.progress.set_fraction(1.0, self)
        self.stop()
        self.emit("scan-finished")
        if Lp().settings.get_value('artist-artwork'):
            Lp().art.cache_artists_info()

    def __scan(self, uris):
        """
            Scan music collection for music files
            @param uris as [string], uris to scan
            @thread safe
        """
        gst_message = None
        if self.__history is None:
            self.__history = History()
        mtimes = Lp().tracks.get_mtimes()
        (new_tracks, new_dirs, ignore_dirs) = self.__get_objects_for_uris(
                                                                         uris)
        orig_tracks = Lp().tracks.get_uris(ignore_dirs)
        was_empty = len(orig_tracks) == 0

        if ignore_dirs:
            if Lp().notify is not None:
                Lp().notify.send(_("Lollypop is detecting an empty folder."),
                                 _("Check your music settings."))
        count = len(new_tracks) + len(orig_tracks)
        # Add monitors on dirs
        if self.__inotify is not None:
            for d in new_dirs:
                self.__inotify.add_monitor(d)

        with SqlCursor(Lp().db) as sql:
            i = 0
            # Look for new files/modified files
            try:
                to_add = []
                for uri in new_tracks:
                    if self.__thread is None:
                        return
                    GLib.idle_add(self.__update_progress, i, count)
                    f = Gio.File.new_for_uri(uri)
                    info = f.query_info('time::modified',
                                        Gio.FileQueryInfoFlags.NONE,
                                        None)
                    mtime = int(info.get_attribute_as_string('time::modified'))
                    # If songs exists and mtime unchanged, continue,
                    # else rescan
                    if uri in orig_tracks:
                        orig_tracks.remove(uri)
                        i += 1
                        if mtime <= mtimes[uri]:
                            i += 1
                            continue
                        else:
                            self.__del_from_db(uri)
                    # On first scan, use modification time
                    # Else, use current time
                    if not was_empty:
                        mtime = int(time())
                    to_add.append((uri, mtime))
                # Clean deleted files
                # Now because we need to populate history
                for uri in orig_tracks:
                    i += 1
                    GLib.idle_add(self.__update_progress, i, count)
                    if uri.startswith('file:'):
                        self.__del_from_db(uri)
                # Add files to db
                for (uri, mtime) in to_add:
                    try:
                        debug("Adding file: %s" % uri)
                        i += 1
                        GLib.idle_add(self.__update_progress, i, count)
                        self.__add2db(uri, mtime)
                    except GLib.GError as e:
                        print("CollectionScanner::__scan:", e)
                        if e.message != gst_message:
                            gst_message = e.message
                            if Lp().notify is not None:
                                Lp().notify.send(gst_message)
                sql.commit()
            except Exception as e:
                print("CollectionScanner::__scan()", e)
        GLib.idle_add(self.__finish)
        del self.__history
        self.__history = None

    def __add2db(self, uri, mtime):
        """
            Add new file to db with informations
            @param uri as string
            @param mtime as int
            @return track id as int
        """
        f = Gio.File.new_for_uri(uri)
        debug("CollectionScanner::add2db(): Read tags")
        info = self.get_info(uri)
        tags = info.get_tags()
        name = f.get_basename()
        title = self.get_title(tags, name)
        artists = self.get_artists(tags)
        composers = self.get_composers(tags)
        performers = self.get_performers(tags)
        a_sortnames = self.get_artist_sortnames(tags)
        aa_sortnames = self.get_album_artist_sortnames(tags)
        album_artists = self.get_album_artist(tags)
        album_name = self.get_album_name(tags)
        genres = self.get_genres(tags)
        discnumber = self.get_discnumber(tags)
        discname = self.get_discname(tags)
        tracknumber = self.get_tracknumber(tags, name)
        year = self.get_year(tags)
        duration = int(info.get_duration()/1000000000)

        # If no artists tag, use album artist
        if artists == '':
            artists = album_artists
        # if artists is always null, no album artists too,
        # use composer/performer
        if artists == '':
            artists = performers
            album_artists = composers
            if artists == '':
                artists = album_artists
            if artists == '':
                artists = _("Unknown")

        debug("CollectionScanner::add2db(): Restore stats")
        # Restore stats
        (track_pop, track_ltime, amtime, album_pop) = self.__history.get(
                                                            name, duration)
        # If nothing in stats, set mtime
        if amtime == 0:
            amtime = mtime

        debug("CollectionScanner::add2db(): Add artists %s" % artists)
        artist_ids = self.add_artists(artists, album_artists, a_sortnames)

        debug("CollectionScanner::add2db(): "
              "Add album artists %s" % album_artists)
        album_artist_ids = self.add_album_artists(album_artists, aa_sortnames)

        new_artist_ids = list(set(album_artist_ids) | set(artist_ids))

        debug("CollectionScanner::add2db(): Add album: "
              "%s, %s" % (album_name, album_artist_ids))
        (album_id, new_album) = self.add_album(album_name, album_artist_ids,
                                               uri, album_pop, amtime, False)

        genre_ids = self.add_genres(genres, album_id)

        # Add track to db
        debug("CollectionScanner::add2db(): Add track")
        track_id = Lp().tracks.add(title, uri, duration,
                                   tracknumber, discnumber, discname,
                                   album_id, year, track_pop,
                                   track_ltime, mtime)

        debug("CollectionScanner::add2db(): Update tracks")
        self.update_track(track_id, artist_ids, genre_ids)
        self.update_album(album_id, album_artist_ids, genre_ids, year)
        if new_album:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
        for genre_id in genre_ids:
            GLib.idle_add(self.emit, 'genre-updated', genre_id, True)
        for artist_id in new_artist_ids:
            GLib.idle_add(self.emit, 'artist-updated', artist_id, True)
        return track_id

    def __del_from_db(self, uri):
        """
            Delete track from db
            @param uri as str
        """
        f = Gio.File.new_for_uri(uri)
        name = f.get_basename()
        track_id = Lp().tracks.get_id_by_uri(uri)
        album_id = Lp().tracks.get_album_id(track_id)
        genre_ids = Lp().tracks.get_genre_ids(track_id)
        album_artist_ids = Lp().albums.get_artist_ids(album_id)
        artist_ids = Lp().tracks.get_artist_ids(track_id)
        popularity = Lp().tracks.get_popularity(track_id)
        ltime = Lp().tracks.get_ltime(track_id)
        mtime = Lp().albums.get_mtime(album_id)
        duration = Lp().tracks.get_duration(track_id)
        album_popularity = Lp().albums.get_popularity(album_id)
        uri = Lp().tracks.get_uri(track_id)
        self.__history.add(name, duration, popularity,
                           ltime, mtime, album_popularity)
        Lp().playlists.remove(uri)
        Lp().tracks.remove(track_id)
        Lp().tracks.clean(track_id)
        deleted = Lp().albums.clean(album_id)
        if deleted:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
            GLib.idle_add(self.emit, 'album-updated', album_id, True)
        for artist_id in album_artist_ids + artist_ids:
            Lp().artists.clean(artist_id)
            GLib.idle_add(self.emit, 'artist-updated', artist_id, False)
        for genre_id in genre_ids:
            Lp().genres.clean(genre_id)
            GLib.idle_add(self.emit, 'genre-updated', genre_id, False)
Ejemplo n.º 4
0
class CollectionScanner(GObject.GObject, TagReader):
    """
        Scan user music collection
    """
    __gsignals__ = {
        'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
        'artist-updated':
        (GObject.SignalFlags.RUN_FIRST, None, (int, int, bool)),
        'genre-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
        'album-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, ))
    }

    def __init__(self):
        """
            Init collection scanner
        """
        GObject.GObject.__init__(self)
        TagReader.__init__(self)

        self.__thread = None
        self.__history = None
        if Lp().settings.get_value('auto-update'):
            self.__inotify = Inotify()
        else:
            self.__inotify = None

    def update(self):
        """
            Update database
        """
        if not self.is_locked():
            paths = Lp().settings.get_music_paths()
            if not paths:
                return

            Lp().window.progress.add(self)
            Lp().window.progress.set_fraction(0.0, self)

            if Lp().notify is not None:
                Lp().notify.send(_("Your music is updating"))
            self.__thread = Thread(target=self.__scan, args=(paths, ))
            self.__thread.daemon = True
            self.__thread.start()

    def is_locked(self):
        """
            Return True if db locked
        """
        return self.__thread is not None and self.__thread.isAlive()

    def stop(self):
        """
            Stop scan
        """
        self.__thread = None

#######################
# PRIVATE             #
#######################

    def __get_objects_for_paths(self, paths):
        """
            Return all tracks/dirs for paths
            @param paths as string
            @return ([tracks path], [dirs path], track count)
        """
        tracks = []
        track_dirs = list(paths)
        for path in paths:
            for root, dirs, files in os.walk(path, followlinks=True):
                # Add dirs
                for d in dirs:
                    track_dirs.append(os.path.join(root, d))
                # Add files
                for name in files:
                    path = os.path.join(root, name)
                    uri = GLib.filename_to_uri(path)
                    try:
                        f = Gio.File.new_for_uri(uri)
                        if is_pls(f):
                            pass
                        elif is_audio(f):
                            tracks.append(uri)
                        else:
                            debug("%s not detected as a music file" % uri)
                    except Exception as e:
                        print(
                            "CollectionScanner::__get_objects_for_paths: %s" %
                            e)
        return (tracks, track_dirs)

    def __update_progress(self, current, total):
        """
            Update progress bar status
            @param scanned items as int, total items as int
        """
        Lp().window.progress.set_fraction(current / total, self)

    def __finish(self):
        """
            Notify from main thread when scan finished
        """
        Lp().window.progress.set_fraction(1.0, self)
        self.stop()
        self.emit("scan-finished")
        if Lp().settings.get_value('artist-artwork'):
            Lp().art.cache_artists_info()

    def __scan(self, paths):
        """
            Scan music collection for music files
            @param paths as [string], paths to scan
            @thread safe
        """
        gst_message = None
        if self.__history is None:
            self.__history = History()
        mtimes = Lp().tracks.get_mtimes()
        orig_tracks = Lp().tracks.get_uris()
        was_empty = len(orig_tracks) == 0

        (new_tracks, new_dirs) = self.__get_objects_for_paths(paths)
        count = len(new_tracks) + len(orig_tracks)
        # Add monitors on dirs
        if self.__inotify is not None:
            for d in new_dirs:
                self.__inotify.add_monitor(d)

        with SqlCursor(Lp().db) as sql:
            i = 0
            for uri in new_tracks:
                if self.__thread is None:
                    return
                GLib.idle_add(self.__update_progress, i, count)
                try:
                    f = Gio.File.new_for_uri(uri)
                    info = f.query_info('time::modified',
                                        Gio.FileQueryInfoFlags.NONE, None)
                    mtime = info.get_attribute_as_string('time::modified')
                    # If songs exists and mtime unchanged, continue,
                    # else rescan
                    if uri in orig_tracks:
                        orig_tracks.remove(uri)
                        i += 1
                        if mtime <= mtimes[uri]:
                            i += 1
                            continue
                        else:
                            self.__del_from_db(uri)
                    info = self.get_info(uri)
                    # On first scan, use modification time
                    # Else, use current time
                    if not was_empty:
                        mtime = int(time())
                    debug("Adding file: %s" % uri)
                    self.__add2db(uri, info, mtime)
                except GLib.GError as e:
                    print(e, uri)
                    if e.message != gst_message:
                        gst_message = e.message
                        if Lp().notify is not None:
                            Lp().notify.send(gst_message)
                except:
                    pass
                i += 1

            # Clean deleted files
            for uri in orig_tracks:
                i += 1
                GLib.idle_add(self.__update_progress, i, count)
                if uri.startswith('file:'):
                    self.__del_from_db(uri)

            sql.commit()
        GLib.idle_add(self.__finish)
        del self.__history
        self.__history = None

    def __add2db(self, uri, info, mtime):
        """
            Add new file to db with informations
            @param uri as string
            @param info as GstPbutils.DiscovererInfo
            @param mtime as int
            @return track id as int
        """
        debug("CollectionScanner::add2db(): Read tags")
        path = GLib.filename_from_uri(uri)[0]
        tags = info.get_tags()
        title = self.get_title(tags, path)
        artists = self.get_artists(tags)
        composers = self.get_composers(tags)
        performers = self.get_performers(tags)
        a_sortnames = self.get_artist_sortnames(tags)
        aa_sortnames = self.get_album_artist_sortnames(tags)
        album_artists = self.get_album_artist(tags)
        album_name = self.get_album_name(tags)
        genres = self.get_genres(tags)
        discnumber = self.get_discnumber(tags)
        discname = self.get_discname(tags)
        tracknumber = self.get_tracknumber(tags, GLib.basename(path))
        year = self.get_year(tags)
        duration = int(info.get_duration() / 1000000000)
        name = GLib.path_get_basename(path)

        # If no artists tag, use album artist
        if artists == '':
            artists = album_artists
        # if artists is always null, no album artists too,
        # use composer/performer
        if artists == '':
            artists = performers
            album_artists = composers
            if artists == '':
                artists = album_artists
            if artists == '':
                artists = _("Unknown")

        debug("CollectionScanner::add2db(): Restore stats")
        # Restore stats
        (track_pop, track_ltime, amtime,
         album_pop) = self.__history.get(name, duration)
        # If nothing in stats, set mtime
        if amtime == 0:
            amtime = mtime
        debug("CollectionScanner::add2db(): Add artists %s" % artists)
        (artist_ids,
         new_artist_ids) = self.add_artists(artists, album_artists,
                                            a_sortnames)
        debug("CollectionScanner::add2db(): "
              "Add album artists %s" % album_artists)
        (album_artist_ids, new_album_artist_ids) = self.add_album_artists(
            album_artists, aa_sortnames)
        new_artist_ids += new_album_artist_ids

        debug("CollectionScanner::add2db(): Add album: "
              "%s, %s" % (album_name, album_artist_ids))
        (album_id, new_album) = self.add_album(album_name, album_artist_ids,
                                               path, album_pop, amtime)

        (genre_ids, new_genre_ids) = self.add_genres(genres, album_id)

        # Add track to db
        debug("CollectionScanner::add2db(): Add track")
        track_id = Lp().tracks.add(title, uri, duration, tracknumber,
                                   discnumber, discname, album_id, year,
                                   track_pop, track_ltime, mtime)

        debug("CollectionScanner::add2db(): Update tracks")
        self.update_track(track_id, artist_ids, genre_ids)
        self.update_album(album_id, album_artist_ids, genre_ids, year)
        # Notify about new artists/genres
        if new_genre_ids or new_artist_ids:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
            for genre_id in new_genre_ids:
                GLib.idle_add(self.emit, 'genre-updated', genre_id, True)
            for artist_id in new_artist_ids:
                GLib.idle_add(self.emit, 'artist-updated', artist_id, album_id,
                              True)
        return track_id

    def __del_from_db(self, uri):
        """
            Delete track from db
            @param uri as str
        """
        path = GLib.filename_from_uri(uri)[0]
        name = GLib.path_get_basename(path)
        track_id = Lp().tracks.get_id_by_uri(uri)
        album_id = Lp().tracks.get_album_id(track_id)
        genre_ids = Lp().tracks.get_genre_ids(track_id)
        album_artist_ids = Lp().albums.get_artist_ids(album_id)
        artist_ids = Lp().tracks.get_artist_ids(track_id)
        popularity = Lp().tracks.get_popularity(track_id)
        ltime = Lp().tracks.get_ltime(track_id)
        mtime = Lp().albums.get_mtime(album_id)
        duration = Lp().tracks.get_duration(track_id)
        album_popularity = Lp().albums.get_popularity(album_id)
        self.__history.add(name, duration, popularity, ltime, mtime,
                           album_popularity)
        Lp().tracks.remove(track_id)
        Lp().tracks.clean(track_id)
        modified = Lp().albums.clean(album_id)
        if modified:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
            GLib.idle_add(self.emit, 'album-updated', album_id)
        for artist_id in album_artist_ids + artist_ids:
            ret = Lp().artists.clean(artist_id)
            if ret:
                GLib.idle_add(self.emit, 'artist-updated', artist_id, album_id,
                              False)
        for genre_id in genre_ids:
            ret = Lp().genres.clean(genre_id)
            if ret:
                GLib.idle_add(self.emit, 'genre-updated', genre_id, False)
Ejemplo n.º 5
0
class CollectionScanner(GObject.GObject, TagReader):
    """
        Scan user music collection
    """
    __gsignals__ = {
        "scan-finished": (GObject.SignalFlags.RUN_FIRST, None, (bool, )),
        "artist-updated": (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
        "genre-updated": (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
        "album-updated": (GObject.SignalFlags.RUN_FIRST, None, (int, bool))
    }

    _WEB_COLLECTION = GLib.get_user_data_dir() + "/lollypop/web_collection"

    def __init__(self):
        """
            Init collection scanner
        """
        GObject.GObject.__init__(self)
        TagReader.__init__(self)

        self.__thread = None
        self.__history = History()
        self.__disable_compilations = True
        if App().settings.get_value("auto-update"):
            self.__inotify = Inotify()
        else:
            self.__inotify = None
        App().albums.update_max_count()
        create_dir(self._WEB_COLLECTION)

    def update(self, scan_type, uris=[]):
        """
            Update database
            @param scan_type as ScanType
            @param uris as [str]
        """
        App().lookup_action("update_db").set_enabled(False)
        # Stop previous scan
        if self.is_locked() and scan_type != ScanType.EPHEMERAL:
            self.stop()
            GLib.timeout_add(250, self.update, scan_type, uris)
        else:
            self.__disable_compilations = not App().settings.get_value(
                "show-compilations")

            if scan_type == ScanType.FULL:
                uris = App().settings.get_music_uris()
            if not uris:
                return
            # Register to progressbar
            if scan_type != ScanType.EPHEMERAL:
                App().window.container.progress.add(self)
                App().window.container.progress.set_fraction(0, self)
            # Launch scan in a separate thread
            self.__thread = Thread(target=self.__scan, args=(scan_type, uris))
            self.__thread.daemon = True
            self.__thread.start()

    def update_album(self, album_id, album_artist_ids, genre_ids, year,
                     timestamp):
        """
            Update album artists based on album-artist and artist tags
            This code auto handle compilations: empty "album artist" with
            different artists
            @param album_id as int
            @param album_artist_ids as [int]
            @param genre_ids as [int]
            @param year as int
            @param timestamp as int
            @commit needed
        """
        if album_artist_ids:
            # Update UI based on previous artist calculation
            mtime = App().albums.get_mtime(album_id)
            if mtime != 0:
                for artist_id in album_artist_ids:
                    GLib.idle_add(self.emit, "artist-updated", artist_id, True)
            App().albums.set_artist_ids(album_id, album_artist_ids)
        # Set artist ids based on content
        else:
            new_album_artist_ids = App().albums.calculate_artist_ids(album_id)
            App().albums.set_artist_ids(album_id, new_album_artist_ids)
        # Update album genres
        for genre_id in genre_ids:
            App().albums.add_genre(album_id, genre_id)
        # Update year based on tracks
        year = App().tracks.get_year_for_album(album_id)
        App().albums.set_year(album_id, year)
        timestamp = App().tracks.get_timestamp_for_album(album_id)
        App().albums.set_timestamp(album_id, timestamp)

    def save_track(self, genres, artists, a_sortnames, mb_artist_id,
                   album_artists, aa_sortnames, mb_album_artist_id, album_name,
                   mb_album_id, uri, album_loved, album_pop, album_rate,
                   album_synced, album_mtime, title, duration, tracknumber,
                   discnumber, discname, year, timestamp, track_mtime,
                   track_pop, track_rate, track_loved, track_ltime,
                   mb_track_id, bpm):
        """
            Add track to DB
            @param genres as str/None
            @param artists as str
            @param a_sortnames as str
            @param mb_artist_id as str
            @param album_artists as str
            @param aa_sortnames as str
            @param mb_album_artist_id as str
            @param album_name as str
            @param mb_album_id as str
            @param uri as str
            @param album_loved as int
            @param album_pop as int
            @param album_rate as int
            @param album_synced as int
            @param album_mtime as int
            @param title as str
            @param duration as int
            @param tracknumber as int
            @param discnumber as int
            @param discname as str
            @param year as int
            @param timestamp as int
            @param track_mtime as int
            @param track_pop as int
            @param track_rate as int
            @param track_loved as int
            @param track_ltime as int
            @param mb_track_id as str
            @param bpm as int
        """
        Logger.debug("CollectionScanner::save_track(): Add artists %s" %
                     artists)
        artist_ids = self.add_artists(artists, a_sortnames, mb_artist_id)

        Logger.debug("CollectionScanner::save_track(): "
                     "Add album artists %s" % album_artists)
        album_artist_ids = self.add_artists(album_artists, aa_sortnames,
                                            mb_album_artist_id)

        # User does not want compilations
        if self.__disable_compilations and not album_artist_ids:
            album_artist_ids = artist_ids

        missing_artist_ids = list(set(album_artist_ids) - set(artist_ids))
        # https://github.com/gnumdk/lollypop/issues/507#issuecomment-200526942
        # Special case for broken tags
        # Can't do more because don't want to break split album behaviour
        if len(missing_artist_ids) == len(album_artist_ids):
            artist_ids += missing_artist_ids

        Logger.debug("CollectionScanner::save_track(): Add album: "
                     "%s, %s" % (album_name, album_artist_ids))
        (album_added,
         album_id) = self.add_album(album_name, mb_album_id, album_artist_ids,
                                    uri, album_loved, album_pop, album_rate,
                                    album_synced, album_mtime)
        if genres is None:
            genre_ids = [Type.WEB]
        else:
            genre_ids = self.add_genres(genres)

        # Add track to db
        Logger.debug("CollectionScanner::save_track(): Add track")
        track_id = App().tracks.add(title, uri, duration, tracknumber,
                                    discnumber, discname, album_id, year,
                                    timestamp, track_pop, track_rate,
                                    track_loved, track_ltime, track_mtime,
                                    mb_track_id, bpm)
        Logger.debug("CollectionScanner::save_track(): Update track")
        self.update_track(track_id, artist_ids, genre_ids)
        Logger.debug("CollectionScanner::save_track(): Update album")
        SqlCursor.commit(App().db)
        self.update_album(album_id, album_artist_ids, genre_ids, year,
                          timestamp)
        SqlCursor.commit(App().db)
        for genre_id in genre_ids:
            # Be sure to not send Type.WEB
            if genre_id >= 0:
                GLib.idle_add(self.emit, "genre-updated", genre_id, True)
        if album_added:
            GLib.idle_add(self.emit, "album-updated", album_id, True)
        return (track_id, album_id)

    def update_track(self, track_id, artist_ids, genre_ids):
        """
            Set track artists/genres
            @param track_id as int
            @param artist_ids as [int]
            @param genre_ids as [int]
            @commit needed
        """
        # Set artists/genres for track
        for artist_id in artist_ids:
            App().tracks.add_artist(track_id, artist_id)
        for genre_id in genre_ids:
            App().tracks.add_genre(track_id, genre_id)

    def del_from_db(self, uri, backup, notify=True):
        """
            Delete track from db
            @param uri as str
            @param backup as bool
            @param notify as bool => send signal about cleanup
            @return (popularity, ltime, mtime,
                     loved album, album_popularity)
        """
        try:
            track_id = App().tracks.get_id_by_uri(uri)
            duration = App().tracks.get_duration(track_id)
            album_id = App().tracks.get_album_id(track_id)
            genre_ids = App().tracks.get_genre_ids(track_id)
            album_artist_ids = App().albums.get_artist_ids(album_id)
            artist_ids = App().tracks.get_artist_ids(track_id)
            track_pop = App().tracks.get_popularity(track_id)
            track_rate = App().tracks.get_rate(track_id)
            track_ltime = App().tracks.get_ltime(track_id)
            album_mtime = App().tracks.get_mtime(track_id)
            track_loved = App().tracks.get_loved(track_id)
            album_pop = App().albums.get_popularity(album_id)
            album_rate = App().albums.get_rate(album_id)
            album_loved = App().albums.get_loved(album_id)
            album_synced = App().albums.get_synced(album_id)
            # Force genre for album
            App().albums.set_genre_ids(album_id, genre_ids)
            if backup:
                f = Gio.File.new_for_uri(uri)
                name = f.get_basename()
                self.__history.add(name, duration, track_pop, track_rate,
                                   track_ltime, album_mtime, track_loved,
                                   album_loved, album_pop, album_rate,
                                   album_synced)
            App().tracks.remove(track_id)
            App().albums.clean()
            App().genres.clean()
            App().artists.clean()
            if notify:
                if App().albums.get_name(album_id) is None:
                    GLib.idle_add(self.emit, "album-updated", album_id, False)
                for artist_id in album_artist_ids + artist_ids:
                    GLib.idle_add(self.emit, "artist-updated", artist_id,
                                  False)
                for genre_id in genre_ids:
                    GLib.idle_add(self.emit, "genre-updated", genre_id, False)
            return (track_pop, track_rate, track_ltime, album_mtime,
                    track_loved, album_loved, album_pop, album_rate)
        except Exception as e:
            Logger.error("CollectionScanner::del_from_db: %s" % e)

    def is_locked(self):
        """
            Return True if db locked
        """
        return self.__thread is not None and self.__thread.isAlive()

    def stop(self):
        """
            Stop scan
        """
        self.__thread = None

    @property
    def inotify(self):
        """
            Get Inotify object
            @return Inotify
        """
        return self.__inotify

#######################
# PRIVATE             #
#######################

    def __update_progress(self, current, total):
        """
            Update progress bar status
            @param scanned items as int, total items as int
        """
        GLib.idle_add(App().window.container.progress.set_fraction,
                      current / total, self)

    def __finish(self, modifications):
        """
            Notify from main thread when scan finished
            @param modifications as bool
        """
        App().lookup_action("update_db").set_enabled(True)
        App().window.container.progress.set_fraction(1.0, self)
        self.stop()
        self.emit("scan-finished", modifications)
        # Update max count value
        App().albums.update_max_count()

    def __add_monitor(self, dirs):
        """
            Monitor any change in a list of directory
            @param dirs as str or list of directory to be monitored
        """
        if self.__inotify is None:
            return
        # Add monitors on dirs
        for d in dirs:
            # Handle a stop request
            if self.__thread is None:
                break
            if d.startswith("file://"):
                self.__inotify.add_monitor(d)

    def __import_web_tracks(self):
        """
            Import locally saved web tracks
        """
        try:
            # Directly add files, walk through directories
            f = Gio.File.new_for_path(self._WEB_COLLECTION)
            infos = f.enumerate_children(SCAN_QUERY_INFO,
                                         Gio.FileQueryInfoFlags.NONE, None)
            for info in infos:
                f = infos.get_child(info)
                if info.get_is_hidden():
                    continue
                elif info.get_file_type() == Gio.FileType.DIRECTORY:
                    pass
                else:
                    (status, content, tag) = f.load_contents()
                    data = json.loads(content)
                    self.save_track(
                        None, ";".join(data["artists"]), "", "", ";".join(
                            data["album_artists"]), "", "", data["album_name"],
                        "", data["uri"], data["album_loved"],
                        data["album_popularity"], data["album_rate"], 0, -1,
                        data["title"], data["duration"], data["tracknumber"],
                        data["discnumber"], data["discname"], data["year"],
                        data["timestamp"], -1, data["track_popularity"],
                        data["track_rate"], data["track_loved"], 0, "", 0)
            infos.close(None)
        except Exception as e:
            Logger.error("CollectionScanner::__import_web_tracks(): %s", e)

    @profile
    def __get_objects_for_uris(self, scan_type, uris):
        """
            Get all tracks and dirs in uris
            @param scan_type as ScanType
            @param uris as string
            @return (tracks [mtimes: int, uri: str], dirs as [uri: str])
        """
        files = []
        dirs = []
        walk_uris = []
        # Check collection exists
        for uri in uris:
            f = Gio.File.new_for_uri(uri)
            if f.query_exists():
                walk_uris.append(uri)
            else:
                return (None, None)

        while walk_uris:
            uri = walk_uris.pop(0)
            try:
                # Directly add files, walk through directories
                f = Gio.File.new_for_uri(uri)
                info = f.query_info(SCAN_QUERY_INFO,
                                    Gio.FileQueryInfoFlags.NONE, None)
                if info.get_file_type() == Gio.FileType.DIRECTORY:
                    dirs.append(uri)
                    infos = f.enumerate_children(SCAN_QUERY_INFO,
                                                 Gio.FileQueryInfoFlags.NONE,
                                                 None)
                    for info in infos:
                        f = infos.get_child(info)
                        child_uri = f.get_uri()
                        if info.get_is_hidden():
                            continue
                        elif info.get_file_type() == Gio.FileType.DIRECTORY:
                            dirs.append(child_uri)
                            walk_uris.append(child_uri)
                        else:
                            mtime = get_mtime(info)
                            files.append((mtime, child_uri))
                    infos.close(None)
                # Only happens if files passed as args
                else:
                    mtime = get_mtime(info)
                    files.append((mtime, uri))
            except Exception as e:
                Logger.error(
                    "CollectionScanner::__get_objects_for_uris(): %s" % e)
        files.sort(reverse=True)
        return (files, dirs)

    @profile
    def __scan(self, scan_type, uris):
        """
            Scan music collection for music files
            @param scan_type as ScanType
            @param uris as [str]
            @thread safe
        """
        if not App().tracks.get_mtimes():
            self.__import_web_tracks()

        (files, dirs) = self.__get_objects_for_uris(scan_type, uris)

        if files is None:
            if App().notify is not None:
                App().notify.send(_("Scan disabled, missing collection"))
            return

        if scan_type == ScanType.NEW_FILES:
            db_uris = App().tracks.get_uris(uris)
        else:
            db_uris = App().tracks.get_uris()
        new_tracks = self.__scan_files(files, db_uris, scan_type)

        if scan_type != ScanType.EPHEMERAL:
            self.__add_monitor(dirs)
            GLib.idle_add(self.__finish, new_tracks)

        if scan_type == ScanType.EPHEMERAL:
            App().player.play_uris(new_tracks)

    def __scan_to_handle(self, uri):
        """
            Check if file has to be handle by scanner
            @param f as Gio.File
            @return bool
        """
        try:
            f = Gio.File.new_for_uri(uri)
            # Scan file
            if App().settings.get_value("import-playlists") and is_pls(f):
                # Handle playlist
                App().playlists.import_tracks(f)
            elif is_audio(f):
                return True
            else:
                Logger.debug("Not detected as a music file: %s" % f.get_uri())
        except Exception as e:
            Logger.error("CollectionScanner::__scan_to_handle(): %s" % e)
        return False

    @profile
    def __scan_files(self, files, db_uris, scan_type):
        """
            Scan music collection for new audio files
            @param files as [str]
            @param db_uris as [str]
            @param scan_type as ScanType
            @return new track uris as [str]
            @thread safe
        """
        SqlCursor.add(App().db)
        i = 0
        # New tracks present in collection
        new_tracks = []
        # Get mtime of all tracks to detect which has to be updated
        db_mtimes = App().tracks.get_mtimes()
        count = len(files) + 1
        try:
            # Scan new files
            for (mtime, uri) in files:
                # Handle a stop request
                if self.__thread is None and scan_type != ScanType.EPHEMERAL:
                    raise Exception("Scan add cancelled")
                try:
                    if not self.__scan_to_handle(uri):
                        continue
                    if mtime > db_mtimes.get(uri, 0):
                        # If not saved, use 0 as mtime, easy delete on quit
                        if scan_type == ScanType.EPHEMERAL:
                            mtime = 0
                        # Do not use mtime if not intial scan
                        elif db_mtimes:
                            mtime = int(time())
                        Logger.debug("Adding file: %s" % uri)
                        self.__add2db(uri, mtime)
                        SqlCursor.allow_thread_execution(App().db)
                        new_tracks.append(uri)
                except Exception as e:
                    Logger.error("CollectionScanner:: __scan_add_files: % s" %
                                 e)
                i += 1
                self.__update_progress(i, count)
            if scan_type != ScanType.EPHEMERAL and self.__thread is not None:
                # We need to check files are always in collections
                if scan_type == ScanType.FULL:
                    collections = App().settings.get_music_uris()
                else:
                    collections = None
                for uri in db_uris:
                    # Handle a stop request
                    if self.__thread is None:
                        raise Exception("Scan del cancelled")
                    in_collection = True
                    if collections is not None:
                        in_collection = False
                        for collection in collections:
                            if collection in uri:
                                in_collection = True
                                break
                    f = Gio.File.new_for_uri(uri)
                    if not in_collection or not f.query_exists():
                        self.del_from_db(uri, True)
                        SqlCursor.allow_thread_execution(App().db)
        except Exception as e:
            Logger.warning("CollectionScanner:: __scan_files: % s" % e)
        SqlCursor.commit(App().db)
        SqlCursor.remove(App().db)
        return new_tracks

    def __add2db(self, uri, track_mtime):
        """
            Add new file(or update one) to db with information
            @param uri as string
            @param track_mtime as int
            @return track id as int
            @warning, be sure SqlCursor is available for App().db
        """
        f = Gio.File.new_for_uri(uri)
        Logger.debug("CollectionScanner::add2db(): Read tags")
        info = self.get_info(uri)
        tags = info.get_tags()
        name = f.get_basename()
        title = self.get_title(tags, name)
        version = self.get_version(tags)
        artists = self.get_artists(tags)
        composers = self.get_composers(tags)
        performers = self.get_performers(tags)
        remixers = self.get_remixers(tags)
        if remixers != "":
            artists += ";%s" % remixers
        a_sortnames = self.get_artist_sortnames(tags)
        aa_sortnames = self.get_album_artist_sortnames(tags)
        album_artists = self.get_album_artists(tags)
        album_name = self.get_album_name(tags)
        album_synced = 0
        mb_album_id = self.get_mb_album_id(tags)
        mb_track_id = self.get_mb_track_id(tags)
        mb_artist_id = self.get_mb_artist_id(tags)
        mb_album_artist_id = self.get_mb_album_artist_id(tags)
        genres = self.get_genres(tags)
        discnumber = self.get_discnumber(tags)
        discname = self.get_discname(tags)
        tracknumber = self.get_tracknumber(tags, name)
        track_popm = self.get_popm(tags)
        bpm = self.get_bpm(tags)
        (year, timestamp) = self.get_original_year(tags)
        if year is None:
            (year, timestamp) = self.get_year(tags)
        duration = int(info.get_duration() / 1000000000)

        if version != "":
            title += " (%s)" % version

        # If no artists tag, use album artist
        if artists == "":
            artists = album_artists
        # if artists is always null, no album artists too,
        # use composer/performer
        if artists == "":
            artists = performers
            album_artists = composers
            if artists == "":
                artists = album_artists
            if artists == "":
                artists = _("Unknown")

        Logger.debug("CollectionScanner::add2db(): Restore stats")
        # Restore stats
        track_id = App().tracks.get_id_by_uri(uri)
        if track_id is None:
            basename = f.get_basename()
            track_id = App().tracks.get_id_by_basename_duration(
                basename, duration)
        if track_id is None:
            (track_pop, track_rate, track_ltime, album_mtime, track_loved,
             album_loved, album_pop, album_rate,
             album_synced) = self.__history.get(name, duration)
        # Delete track and restore from it
        else:
            (track_pop, track_rate, track_ltime, album_mtime, track_loved,
             album_loved, album_pop,
             album_rate) = self.del_from_db(uri, False)
        # Prefer popm to internal rate
        if track_popm != 0:
            track_rate = track_popm
        # If nothing in stats, use track mtime
        if album_mtime == 0:
            album_mtime = track_mtime

        (track_id, album_id) = self.save_track(
            genres, artists, a_sortnames, mb_artist_id, album_artists,
            aa_sortnames, mb_album_artist_id, album_name, mb_album_id, uri,
            album_loved, album_pop, album_rate, album_synced, album_mtime,
            title, duration, tracknumber, discnumber, discname, year,
            timestamp, track_mtime, track_pop, track_rate, track_loved,
            track_ltime, mb_track_id, bpm)
        return track_id
Ejemplo n.º 6
0
class CollectionScanner(GObject.GObject, ScannerTagReader):
    """
        Scan user music collection
    """
    __gsignals__ = {
        'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
        'artist-added': (GObject.SignalFlags.RUN_FIRST, None, (int, int)),
        'genre-added': (GObject.SignalFlags.RUN_FIRST, None, (int,)),
        'album-update': (GObject.SignalFlags.RUN_FIRST, None, (int,))
    }

    def __init__(self):
        """
            Init collection scanner
        """
        GObject.GObject.__init__(self)
        ScannerTagReader.__init__(self)

        self._thread = None
        self._inotify = None
        self._history = None
        if Lp().settings.get_value('auto-update'):
            self._inotify = Inotify()
        self._progress = None

    def update(self, progress):
        """
            Update database
            @param progress as Gtk.Scale
        """
        if not self.is_locked():
            progress.show()
            self._progress = progress
            # Keep track of on file with missing codecs
            self._missing_codecs = None
            self.init_discover()
            paths = Lp().settings.get_music_paths()
            if not paths:
                return

            if Lp().notify is not None:
                Lp().notify.send(_("Your music is updating"))
            self._thread = Thread(target=self._scan, args=(paths,))
            self._thread.daemon = True
            self._thread.start()

    def is_locked(self):
        """
            Return True if db locked
        """
        return self._thread is not None and self._thread.isAlive()

    def stop(self):
        """
            Stop scan
        """
        self._thread = None
        if self._progress is not None:
            self._progress.hide()
            self._progress.set_fraction(0.0)
            self._progress = None

#######################
# PRIVATE             #
#######################
    def _get_objects_for_paths(self, paths):
        """
            Return all tracks/dirs for paths
            @param paths as string
            @return ([tracks path], [dirs path], track count)
        """
        tracks = []
        track_dirs = list(paths)
        count = 0
        for path in paths:
            for root, dirs, files in os.walk(path):
                # Add dirs
                for d in dirs:
                    track_dirs.append(os.path.join(root, d))
                # Add files
                for name in files:
                    filepath = os.path.join(root, name)
                    try:
                        f = Gio.File.new_for_path(filepath)
                        if is_pls(f):
                            pass
                        elif is_audio(f):
                            tracks.append(filepath)
                            count += 1
                        else:
                            debug("%s not detected as a music file" % filepath)
                    except Exception as e:
                        print("CollectionScanner::_get_objects_for_paths: %s"
                              % e)
        return (tracks, track_dirs, count)

    def _update_progress(self, current, total):
        """
            Update progress bar status
            @param scanned items as int, total items as int
        """
        if self._progress is not None:
            self._progress.set_fraction(current/total)

    def _finish(self):
        """
            Notify from main thread when scan finished
        """
        self.stop()
        self.emit("scan-finished")
        if Lp().settings.get_value('artist-artwork'):
            Lp().art.cache_artists_art()
        if self._missing_codecs is not None:
            Lp().player.load_external(
                                    GLib.filename_to_uri(self._missing_codecs))
            Lp().player.play_first_external()

    def _scan(self, paths):
        """
            Scan music collection for music files
            @param paths as [string], paths to scan
            @thread safe
        """
        if self._history is None:
            self._history = History()
        mtimes = Lp().tracks.get_mtimes()
        orig_tracks = Lp().tracks.get_paths()
        was_empty = len(orig_tracks) == 0

        (new_tracks, new_dirs, count) = self._get_objects_for_paths(paths)
        count += len(orig_tracks)

        # Add monitors on dirs
        if self._inotify is not None:
            for d in new_dirs:
                self._inotify.add_monitor(d)

        with SqlCursor(Lp().db) as sql:
            i = 0
            for filepath in new_tracks:
                if self._thread is None:
                    return
                GLib.idle_add(self._update_progress, i, count)
                try:
                    # If songs exists and mtime unchanged, continue,
                    # else rescan
                    if filepath in orig_tracks:
                        orig_tracks.remove(filepath)
                        i += 1
                        mtime = int(os.path.getmtime(filepath))
                        if mtime <= mtimes[filepath]:
                            i += 1
                            continue
                        else:
                            self._del_from_db(filepath)
                    infos = self.get_infos(filepath)
                    # On first scan, use modification time
                    # Else, use current time
                    if was_empty:
                        mtime = int(os.path.getmtime(filepath))
                    else:
                        mtime = int(time())
                    debug("Adding file: %s" % filepath)
                    self._add2db(filepath, infos, mtime)
                except Exception as e:
                    debug("Error scanning: %s, %s" % (filepath, e))
                    string = "%s" % e
                    if string.startswith('gst-core-error-quark'):
                        self._missing_codecs = filepath
                i += 1

            # Clean deleted files
            for filepath in orig_tracks:
                i += 1
                GLib.idle_add(self._update_progress, i, count)
                self._del_from_db(filepath)

            sql.commit()
        GLib.idle_add(self._finish)
        del self._history
        self._history = None

    def _add2db(self, filepath, infos, mtime):
        """
            Add new file to db with informations
            @param filepath as string
            @param infos as GstPbutils.DiscovererInfo
            @param mtime as int
            @return track id as int
        """
        debug("CollectionScanner::add2db(): Read tags")
        tags = infos.get_tags()

        title = self.get_title(tags, filepath)
        artists = self.get_artists(tags)
        composers = self.get_composers(tags)
        performers = self.get_performers(tags)
        a_sortnames = self.get_artist_sortnames(tags)
        aa_sortnames = self.get_album_artist_sortnames(tags)
        album_artists = self.get_album_artist(tags)
        album_name = self.get_album_name(tags)
        genres = self.get_genres(tags)
        discnumber = self.get_discnumber(tags)
        discname = self.get_discname(tags)
        tracknumber = self.get_tracknumber(tags)
        year = self.get_year(tags)
        duration = int(infos.get_duration()/1000000000)
        name = GLib.path_get_basename(filepath)

        # If no artists tag, use album artist
        if artists == '':
            artists = album_artists
        # if artists is always null, no album artists too,
        # use composer/performer
        if artists == '':
            artists = performers
            album_artists = composers
            if artists == '':
                artists = album_artists
            if artists == '':
                artists = _("Unknown")

        debug("CollectionScanner::add2db(): Restore stats")
        # Restore stats
        (track_pop, track_ltime, amtime, album_pop) = self._history.get(
                                                            name, duration)
        # If nothing in stats, set mtime
        if amtime == 0:
            amtime = mtime
        debug("CollectionScanner::add2db(): Add artists %s" % artists)
        (artist_ids, new_artist_ids) = self.add_artists(artists,
                                                        album_artists,
                                                        a_sortnames)
        debug("CollectionScanner::add2db(): "
              "Add album artists %s" % album_artists)
        (album_artist_ids, new_album_artist_ids) = self.add_album_artists(
                                                                 album_artists,
                                                                 aa_sortnames)
        new_artist_ids += new_album_artist_ids

        debug("CollectionScanner::add2db(): Add album: "
              "%s, %s, %s" % (album_name, album_artist_ids, year))
        (album_id, new_album) = self.add_album(album_name, album_artist_ids,
                                               year, filepath,
                                               album_pop, amtime)

        (genre_ids, new_genre_ids) = self.add_genres(genres, album_id)

        # Add track to db
        debug("CollectionScanner::add2db(): Add track")
        track_id = Lp().tracks.add(title, filepath, duration,
                                   tracknumber, discnumber, discname,
                                   album_id, year, track_pop,
                                   track_ltime, mtime)

        debug("CollectionScanner::add2db(): Update tracks")
        self.update_track(track_id, artist_ids, genre_ids)
        self.update_album(album_id, album_artist_ids, genre_ids)
        # Notify about new artists/genres
        if new_genre_ids or new_artist_ids:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
            for genre_id in new_genre_ids:
                GLib.idle_add(self.emit, 'genre-added', genre_id)
            for artist_id in new_artist_ids:
                GLib.idle_add(self.emit, 'artist-added', artist_id, album_id)
        return track_id

    def _del_from_db(self, filepath):
        """
            Delete track from db
            @param filepath as str
        """
        name = GLib.path_get_basename(filepath)
        track_id = Lp().tracks.get_id_by_path(filepath)
        album_id = Lp().tracks.get_album_id(track_id)
        genre_ids = Lp().tracks.get_genre_ids(track_id)
        album_artist_ids = Lp().albums.get_artist_ids(album_id)
        artist_ids = Lp().tracks.get_artist_ids(track_id)
        popularity = Lp().tracks.get_popularity(track_id)
        ltime = Lp().tracks.get_ltime(track_id)
        mtime = Lp().albums.get_mtime(album_id)
        duration = Lp().tracks.get_duration(track_id)
        album_popularity = Lp().albums.get_popularity(album_id)
        self._history.add(name, duration, popularity,
                          ltime, mtime, album_popularity)
        Lp().tracks.remove(track_id)
        Lp().tracks.clean(track_id)
        modified = Lp().albums.clean(album_id)
        if modified:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
            GLib.idle_add(self.emit, 'album-update', album_id)
        for artist_id in album_artist_ids + artist_ids:
            Lp().artists.clean(artist_id)
        for genre_id in genre_ids:
            Lp().genres.clean(genre_id)
Ejemplo n.º 7
0
class CollectionScanner(GObject.GObject, ScannerTagReader):
    """
        Scan user music collection
    """
    __gsignals__ = {
        'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
        'artist-added': (GObject.SignalFlags.RUN_FIRST, None, (int, int)),
        'genre-added': (GObject.SignalFlags.RUN_FIRST, None, (int, )),
        'album-update': (GObject.SignalFlags.RUN_FIRST, None, (int, ))
    }

    def __init__(self):
        """
            Init collection scanner
        """
        GObject.GObject.__init__(self)
        ScannerTagReader.__init__(self)

        self._thread = None
        self._inotify = None
        self._history = None
        if Lp().settings.get_value('auto-update'):
            self._inotify = Inotify()
        self._progress = None

    def update(self, progress):
        """
            Update database
            @param progress as Gtk.Scale
        """
        if not self.is_locked():
            progress.show()
            self._progress = progress
            # Keep track of on file with missing codecs
            self._missing_codecs = None
            self.init_discover()
            paths = Lp().settings.get_music_paths()
            if not paths:
                return

            if Lp().notify is not None:
                Lp().notify.send(_("Your music is updating"))
            self._thread = Thread(target=self._scan, args=(paths, ))
            self._thread.daemon = True
            self._thread.start()

    def is_locked(self):
        """
            Return True if db locked
        """
        return self._thread is not None and self._thread.isAlive()

    def stop(self):
        """
            Stop scan
        """
        self._thread = None
        if self._progress is not None:
            self._progress.hide()
            self._progress.set_fraction(0.0)
            self._progress = None

#######################
# PRIVATE             #
#######################

    def _get_objects_for_paths(self, paths):
        """
            Return all tracks/dirs for paths
            @param paths as string
            @return ([tracks path], [dirs path], track count)
        """
        tracks = []
        track_dirs = list(paths)
        count = 0
        for path in paths:
            for root, dirs, files in os.walk(path):
                # Add dirs
                for d in dirs:
                    track_dirs.append(os.path.join(root, d))
                # Add files
                for name in files:
                    filepath = os.path.join(root, name)
                    try:
                        f = Gio.File.new_for_path(filepath)
                        if is_pls(f):
                            pass
                        elif is_audio(f):
                            tracks.append(filepath)
                            count += 1
                        else:
                            debug("%s not detected as a music file" % filepath)
                    except Exception as e:
                        print("CollectionScanner::_get_objects_for_paths: %s" %
                              e)
        return (tracks, track_dirs, count)

    def _update_progress(self, current, total):
        """
            Update progress bar status
            @param scanned items as int, total items as int
        """
        if self._progress is not None:
            self._progress.set_fraction(current / total)

    def _finish(self):
        """
            Notify from main thread when scan finished
        """
        self.stop()
        self.emit("scan-finished")
        if self._missing_codecs is not None:
            Lp().player.load_external(
                GLib.filename_to_uri(self._missing_codecs))
            Lp().player.play_first_external()

    def _scan(self, paths):
        """
            Scan music collection for music files
            @param paths as [string], paths to scan
            @thread safe
        """
        if self._history is None:
            self._history = History()
        mtimes = Lp().tracks.get_mtimes()
        orig_tracks = Lp().tracks.get_paths()
        was_empty = len(orig_tracks) == 0

        (new_tracks, new_dirs, count) = self._get_objects_for_paths(paths)
        count += len(orig_tracks)

        # Add monitors on dirs
        if self._inotify is not None:
            for d in new_dirs:
                self._inotify.add_monitor(d)

        with SqlCursor(Lp().db) as sql:
            i = 0
            for filepath in new_tracks:
                if self._thread is None:
                    return
                GLib.idle_add(self._update_progress, i, count)
                try:
                    # If songs exists and mtime unchanged, continue,
                    # else rescan
                    if filepath in orig_tracks:
                        orig_tracks.remove(filepath)
                        i += 1
                        mtime = int(os.path.getmtime(filepath))
                        if mtime <= mtimes[filepath]:
                            i += 1
                            continue
                        else:
                            self._del_from_db(filepath)
                    infos = self.get_infos(filepath)
                    # On first scan, use modification time
                    # Else, use current time
                    if was_empty:
                        mtime = int(os.path.getmtime(filepath))
                    else:
                        mtime = int(time())
                    debug("Adding file: %s" % filepath)
                    self._add2db(filepath, infos, mtime)
                except Exception as e:
                    debug("Error scanning: %s, %s" % (filepath, e))
                    string = "%s" % e
                    if string.startswith('gst-core-error-quark'):
                        self._missing_codecs = filepath
                i += 1

            # Clean deleted files
            for filepath in orig_tracks:
                i += 1
                GLib.idle_add(self._update_progress, i, count)
                self._del_from_db(filepath)

            sql.commit()
        GLib.idle_add(self._finish)
        del self._history
        self._history = None

    def _add2db(self, filepath, infos, mtime):
        """
            Add new file to db with informations
            @param filepath as string
            @param infos as GstPbutils.DiscovererInfo
            @param mtime as int
            @return track id as int
        """
        debug("CollectionScanner::add2db(): Read tags")
        tags = infos.get_tags()

        title = self.get_title(tags, filepath)
        artists = self.get_artists(tags)
        composers = self.get_composers(tags)
        performers = self.get_performers(tags)
        sortnames = self.get_artist_sortnames(tags)
        album_artists = self.get_album_artist(tags)
        album_name = self.get_album_name(tags)
        genres = self.get_genres(tags)
        discnumber = self.get_discnumber(tags)
        tracknumber = self.get_tracknumber(tags)
        year = self.get_year(tags)
        duration = int(infos.get_duration() / 1000000000)
        name = GLib.path_get_basename(filepath)

        # If no artists tag, use album artist
        if artists == '':
            artists = album_artists
        # if artists is always null, no album artists too,
        # use composer/performer
        if artists == '':
            artists = performers
            album_artists = composers
            if artists == '':
                artists = album_artists

        debug("CollectionScanner::add2db(): Restore stats")
        # Restore stats
        (track_pop, track_ltime, amtime,
         album_pop) = self._history.get(name, duration)
        # If nothing in stats, set mtime
        if amtime == 0:
            amtime = mtime
        debug("CollectionScanner::add2db(): Add artists %s" % artists)
        (artist_ids,
         new_artist_ids) = self.add_artists(artists, album_artists, sortnames)
        debug("CollectionScanner::add2db(): "
              "Add album artists %s" % album_artists)
        (album_artist_ids,
         new_album_artist_ids) = self.add_album_artists(album_artists)
        new_artist_ids += new_album_artist_ids

        # Check for album artist, if none, use first available artist
        no_album_artist = False
        if not album_artist_ids:
            album_artist_ids = artist_ids
            no_album_artist = True
        debug("CollectionScanner::add2db(): Add album: "
              "%s, %s, %s" % (album_name, album_artist_ids, year))
        (album_id, new_album) = self.add_album(album_name, album_artist_ids,
                                               no_album_artist, year, filepath,
                                               album_pop, amtime)

        (genre_ids, new_genre_ids) = self.add_genres(genres, album_id)

        # Add track to db
        debug("CollectionScanner::add2db(): Add track")
        track_id = Lp().tracks.add(title, filepath, duration, tracknumber,
                                   discnumber, album_id, year, track_pop,
                                   track_ltime, mtime)

        debug("CollectionScanner::add2db(): Update tracks")
        self.update_album(album_id, album_artist_ids, genre_ids)
        self.update_track(track_id, artist_ids, genre_ids)
        # Notify about new artists/genres
        if new_genre_ids or new_artist_ids:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
            for genre_id in new_genre_ids:
                GLib.idle_add(self.emit, 'genre-added', genre_id)
            for artist_id in new_artist_ids:
                GLib.idle_add(self.emit, 'artist-added', artist_id, album_id)
        return track_id

    def _del_from_db(self, filepath):
        """
            Delete track from db
            @param filepath as str
        """
        name = GLib.path_get_basename(filepath)
        track_id = Lp().tracks.get_id_by_path(filepath)
        album_id = Lp().tracks.get_album_id(track_id)
        genre_ids = Lp().tracks.get_genre_ids(track_id)
        album_artist_ids = Lp().albums.get_artist_ids(album_id)
        artist_ids = Lp().tracks.get_artist_ids(track_id)
        popularity = Lp().tracks.get_popularity(track_id)
        ltime = Lp().tracks.get_ltime(track_id)
        mtime = Lp().albums.get_mtime(album_id)
        duration = Lp().tracks.get_duration(track_id)
        album_popularity = Lp().albums.get_popularity(album_id)
        self._history.add(name, duration, popularity, ltime, mtime,
                          album_popularity)
        Lp().tracks.remove(track_id)
        Lp().tracks.clean(track_id)
        modified = Lp().albums.clean(album_id)
        if modified:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
            GLib.idle_add(self.emit, 'album-update', album_id)
        for artist_id in album_artist_ids + artist_ids:
            Lp().artists.clean(artist_id)
        for genre_id in genre_ids:
            Lp().genres.clean(genre_id)
Ejemplo n.º 8
0
class CollectionScanner(GObject.GObject, TagReader):
    """
        Scan user music collection
    """
    __gsignals__ = {
        'scan-finished': (GObject.SignalFlags.RUN_FIRST, None, ()),
        'artist-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
        'genre-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool)),
        'album-updated': (GObject.SignalFlags.RUN_FIRST, None, (int, bool))
    }

    def __init__(self):
        """
            Init collection scanner
        """
        GObject.GObject.__init__(self)
        TagReader.__init__(self)

        self.__thread = None
        self.__history = None
        if Lp().settings.get_value('auto-update'):
            self.__inotify = Inotify()
        else:
            self.__inotify = None
        Lp().albums.update_max_count()

    def update(self):
        """
            Update database
        """
        if not self.is_locked():
            uris = Lp().settings.get_music_uris()
            if not uris:
                return

            Lp().window.progress.add(self)
            Lp().window.progress.set_fraction(0.0, self)

            self.__thread = Thread(target=self.__scan, args=(uris,))
            self.__thread.daemon = True
            self.__thread.start()

    def clean_charts(self):
        """
            Clean charts in db
        """
        self.__thread = Thread(target=self.__clean_charts)
        self.__thread.daemon = True
        self.__thread.start()

    def is_locked(self):
        """
            Return True if db locked
        """
        return self.__thread is not None and self.__thread.isAlive()

    def stop(self):
        """
            Stop scan
        """
        self.__thread = None

#######################
# PRIVATE             #
#######################
    def __clean_charts(self):
        """
            Clean charts in db
        """
        track_ids = Lp().tracks.get_old_charts_track_ids(int(time()))
        Lp().db.del_tracks(track_ids)
        self.stop()

    def __get_objects_for_uris(self, uris):
        """
            Return all tracks/dirs for uris
            @param uris as string
            @return (track uri as [str], track dirs as [str],
                     ignore dirs as [str])
        """
        tracks = []
        ignore_dirs = []
        track_dirs = list(uris)
        walk_uris = list(uris)
        while walk_uris:
            uri = walk_uris.pop(0)
            empty = True
            try:
                d = Lio.File.new_for_uri(uri)
                infos = d.enumerate_children(
                    'standard::name,standard::type,standard::is-hidden',
                    Gio.FileQueryInfoFlags.NONE,
                    None)
            except Exception as e:
                print("CollectionScanner::__get_objects_for_uris():", e)
                continue
            for info in infos:
                f = infos.get_child(info)
                child_uri = f.get_uri()
                empty = False
                if info.get_is_hidden():
                    continue
                elif info.get_file_type() == Gio.FileType.DIRECTORY:
                    track_dirs.append(child_uri)
                    walk_uris.append(child_uri)
                else:
                    try:
                        f = Lio.File.new_for_uri(child_uri)
                        if is_pls(f):
                            pass
                        elif is_audio(f):
                            tracks.append(child_uri)
                        else:
                            debug("%s not detected as a music file" % uri)
                    except Exception as e:
                        print("CollectionScanner::"
                              "__get_objects_for_uris():", e)
            # If a root uri is empty
            # Ensure user is not doing something bad
            if empty and uri in uris:
                ignore_dirs.append(uri)
        return (tracks, track_dirs, ignore_dirs)

    def __update_progress(self, current, total):
        """
            Update progress bar status
            @param scanned items as int, total items as int
        """
        Lp().window.progress.set_fraction(current / total, self)

    def __finish(self):
        """
            Notify from main thread when scan finished
        """
        Lp().window.progress.set_fraction(1.0, self)
        self.stop()
        self.emit("scan-finished")
        # Update max count value
        Lp().albums.update_max_count()
        if Lp().settings.get_value('artist-artwork'):
            Lp().art.cache_artists_info()

    def __scan(self, uris):
        """
            Scan music collection for music files
            @param uris as [string], uris to scan
            @thread safe
        """
        gst_message = None
        if self.__history is None:
            self.__history = History()
        mtimes = Lp().tracks.get_mtimes()
        (new_tracks, new_dirs, ignore_dirs) = self.__get_objects_for_uris(
                                                                         uris)
        orig_tracks = Lp().tracks.get_uris(ignore_dirs)
        was_empty = len(orig_tracks) == 0

        if ignore_dirs:
            if Lp().notify is not None:
                Lp().notify.send(_("Lollypop is detecting an empty folder."),
                                 _("Check your music settings."))
        count = len(new_tracks) + len(orig_tracks)
        # Add monitors on dirs
        if self.__inotify is not None:
            for d in new_dirs:
                self.__inotify.add_monitor(d)

        with SqlCursor(Lp().db) as sql:
            i = 0
            # Look for new files/modified files
            try:
                to_add = []
                for uri in new_tracks:
                    if self.__thread is None:
                        return
                    GLib.idle_add(self.__update_progress, i, count)
                    f = Lio.File.new_for_uri(uri)
                    info = f.query_info('time::modified',
                                        Gio.FileQueryInfoFlags.NONE,
                                        None)
                    mtime = int(info.get_attribute_as_string('time::modified'))
                    # If songs exists and mtime unchanged, continue,
                    # else rescan
                    if uri in orig_tracks:
                        orig_tracks.remove(uri)
                        i += 1
                        if mtime <= mtimes[uri]:
                            i += 1
                            continue
                        else:
                            self.__del_from_db(uri)
                    # On first scan, use modification time
                    # Else, use current time
                    if not was_empty:
                        mtime = int(time())
                    to_add.append((uri, mtime))
                # Clean deleted files
                # Now because we need to populate history
                for uri in orig_tracks:
                    i += 1
                    GLib.idle_add(self.__update_progress, i, count)
                    if uri.startswith('file:'):
                        self.__del_from_db(uri)
                # Add files to db
                for (uri, mtime) in to_add:
                    try:
                        debug("Adding file: %s" % uri)
                        i += 1
                        GLib.idle_add(self.__update_progress, i, count)
                        self.__add2db(uri, mtime)
                    except GLib.GError as e:
                        print("CollectionScanner::__scan:", e, uri)
                        if e.message != gst_message:
                            gst_message = e.message
                            if Lp().notify is not None:
                                Lp().notify.send(gst_message, uri)
                sql.commit()
            except Exception as e:
                print("CollectionScanner::__scan()", e)
        GLib.idle_add(self.__finish)
        del self.__history
        self.__history = None

    def __add2db(self, uri, mtime):
        """
            Add new file to db with informations
            @param uri as string
            @param mtime as int
            @return track id as int
        """
        f = Lio.File.new_for_uri(uri)
        debug("CollectionScanner::add2db(): Read tags")
        info = self.get_info(uri)
        tags = info.get_tags()
        name = f.get_basename()
        title = self.get_title(tags, name)
        artists = self.get_artists(tags)
        composers = self.get_composers(tags)
        performers = self.get_performers(tags)
        a_sortnames = self.get_artist_sortnames(tags)
        aa_sortnames = self.get_album_artist_sortnames(tags)
        album_artists = self.get_album_artist(tags)
        album_name = self.get_album_name(tags)
        genres = self.get_genres(tags)
        discnumber = self.get_discnumber(tags)
        discname = self.get_discname(tags)
        tracknumber = self.get_tracknumber(tags, name)
        year = self.get_original_year(tags)
        if year is None:
            year = self.get_year(tags)
        duration = int(info.get_duration()/1000000000)

        # If no artists tag, use album artist
        if artists == '':
            artists = album_artists
        # if artists is always null, no album artists too,
        # use composer/performer
        if artists == '':
            artists = performers
            album_artists = composers
            if artists == '':
                artists = album_artists
            if artists == '':
                artists = _("Unknown")

        debug("CollectionScanner::add2db(): Restore stats")
        # Restore stats
        (track_pop, track_rate, track_ltime, amtime,
         loved, album_pop, album_rate) = self.__history.get(name, duration)
        # If nothing in stats, use track mtime
        if amtime == 0:
            amtime = mtime

        debug("CollectionScanner::add2db(): Add artists %s" % artists)
        artist_ids = self.add_artists(artists, album_artists, a_sortnames)

        debug("CollectionScanner::add2db(): "
              "Add album artists %s" % album_artists)
        album_artist_ids = self.add_album_artists(album_artists, aa_sortnames)

        new_artist_ids = list(set(album_artist_ids) | set(artist_ids))

        debug("CollectionScanner::add2db(): Add album: "
              "%s, %s" % (album_name, album_artist_ids))
        (album_id, new_album) = self.add_album(album_name, album_artist_ids,
                                               uri, loved, album_pop,
                                               album_rate, False)

        genre_ids = self.add_genres(genres)

        # Add track to db
        debug("CollectionScanner::add2db(): Add track")
        track_id = Lp().tracks.add(title, uri, duration,
                                   tracknumber, discnumber, discname,
                                   album_id, year, track_pop, track_rate,
                                   track_ltime)

        debug("CollectionScanner::add2db(): Update tracks")
        self.update_track(track_id, artist_ids, genre_ids, mtime)
        self.update_album(album_id, album_artist_ids, genre_ids, amtime, year)
        if new_album:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
        for genre_id in genre_ids:
            GLib.idle_add(self.emit, 'genre-updated', genre_id, True)
        for artist_id in new_artist_ids:
            GLib.idle_add(self.emit, 'artist-updated', artist_id, True)
        return track_id

    def __del_from_db(self, uri):
        """
            Delete track from db
            @param uri as str
        """
        f = Lio.File.new_for_uri(uri)
        name = f.get_basename()
        track_id = Lp().tracks.get_id_by_uri(uri)
        album_id = Lp().tracks.get_album_id(track_id)
        genre_ids = Lp().tracks.get_genre_ids(track_id)
        album_artist_ids = Lp().albums.get_artist_ids(album_id)
        artist_ids = Lp().tracks.get_artist_ids(track_id)
        popularity = Lp().tracks.get_popularity(track_id)
        rate = Lp().tracks.get_rate(track_id)
        ltime = Lp().tracks.get_ltime(track_id)
        mtime = Lp().albums.get_mtime(album_id)
        duration = Lp().tracks.get_duration(track_id)
        album_popularity = Lp().albums.get_popularity(album_id)
        album_rate = Lp().albums.get_rate(album_id)
        loved = Lp().albums.get_loved(album_id)
        uri = Lp().tracks.get_uri(track_id)
        self.__history.add(name, duration, popularity, rate,
                           ltime, mtime, loved, album_popularity, album_rate)
        Lp().tracks.remove(track_id)
        Lp().tracks.clean(track_id)
        deleted = Lp().albums.clean(album_id)
        if deleted:
            with SqlCursor(Lp().db) as sql:
                sql.commit()
            GLib.idle_add(self.emit, 'album-updated', album_id, True)
        for artist_id in album_artist_ids + artist_ids:
            Lp().artists.clean(artist_id)
            GLib.idle_add(self.emit, 'artist-updated', artist_id, False)
        for genre_id in genre_ids:
            Lp().genres.clean(genre_id)
            GLib.idle_add(self.emit, 'genre-updated', genre_id, False)