Esempio n. 1
0
    def _createDatabaseModelFromSearchModel(self, artist, title, sr):
        """
        Take the given SearchResult Release Model and create a Database Model
        :type artist: Artist
        :type title: str
        :type sr: searchEngines.models.Release.Release
        """
        createDattabaseModelFromSearchModelRelease = Release()
        printableTitle = title.encode('ascii', 'ignore').decode('utf-8')
        releaseByExternalIds = self._getFromDatabaseByExternalIds(sr.musicBrainzId,
                                                                  sr.iTunesId,
                                                                  sr.lastFMId,
                                                                  sr.amgId,
                                                                  sr.spotifyId)
        if releaseByExternalIds:
            if not releaseByExternalIds.alternateNames:
                releaseByExternalIds.alternateNames = []
            if title not in releaseByExternalIds.alternateNames:
                self.logger.debug("Found Title By External Ids [" +
                                  releaseByExternalIds.title.encode('ascii', 'ignore')
                                  .decode('utf-8') + "] Added [" +
                                  printableTitle + "] To AlternateNames")
                if not releaseByExternalIds.alternateNames:
                    releaseByExternalIds.alternateNames = []
                releaseByExternalIds.alternateNames.append(title)
                releaseByExternalIds.lastUpdated = arrow.utcnow().datetime
                self.session.commit()
            return releaseByExternalIds
        createDattabaseModelFromSearchModelRelease.artist = artist
        createDattabaseModelFromSearchModelRelease.roadieId = sr.roadieId
        createDattabaseModelFromSearchModelRelease.title = title
        createDattabaseModelFromSearchModelRelease.releaseDate = parseDate(sr.releaseDate)
        createDattabaseModelFromSearchModelRelease.trackCount = sr.trackCount
        createDattabaseModelFromSearchModelRelease.mediaCount = sr.mediaCount
        createDattabaseModelFromSearchModelRelease.thumbnail = sr.thumbnail
        createDattabaseModelFromSearchModelRelease.profile = sr.profile
        if sr.releaseType == SearchReleaseType.Album:
            createDattabaseModelFromSearchModelRelease.releaseType = 'Album'
        elif sr.releaseType == SearchReleaseType.EP:
            createDattabaseModelFromSearchModelRelease.releaseType = 'EP'
        elif sr.releaseType == SearchReleaseType.Single:
            createDattabaseModelFromSearchModelRelease.releaseType = 'Single'
        createDattabaseModelFromSearchModelRelease.iTunesId = sr.iTunesId
        createDattabaseModelFromSearchModelRelease.amgId = sr.amgId
        createDattabaseModelFromSearchModelRelease.lastFMId = sr.lastFMId
        createDattabaseModelFromSearchModelRelease.lastFMSummary = sr.lastFMSummary
        createDattabaseModelFromSearchModelRelease.musicBrainzId = sr.musicBrainzId
        createDattabaseModelFromSearchModelRelease.spotifyId = sr.spotifyId
        createDattabaseModelFromSearchModelRelease.amgId = sr.amgId
        createDattabaseModelFromSearchModelRelease.tags = sr.tags
        createDattabaseModelFromSearchModelRelease.alternateNames = sr.alternateNames

        createDattabaseModelFromSearchModelRelease.urls = sr.urls
        if sr.images:
            createDattabaseModelFromSearchModelReleaseimages = []
            for image in sr.images:
                if image.image:
                    i = Image()
                    i.roadieId = image.roadieId
                    i.url = image.url
                    i.caption = image.caption
                    i.image = image.image
                    i.signature = image.signature
                    createDattabaseModelFromSearchModelReleaseimages.append(i)
            createDattabaseModelFromSearchModelRelease.images = createDattabaseModelFromSearchModelReleaseimages
            self.logger.debug(
                "= Added [" + str(len(createDattabaseModelFromSearchModelRelease.images)) + "] Images to Release")

        # TODO
        # See if cover file found in Release Folder
        # coverFile = os.path.join(mp3Folder, "cover.jpg")
        # if os.path.isfile(coverFile):
        #     ba = self.readImageThumbnailBytesFromFile(coverFile)
        # else:
        #     coverFile = os.path.join(mp3Folder, "front.jpg")
        #     if os.path.isfile(coverFile):
        #         ba = self.readImageThumbnailBytesFromFile(coverFile)
        # # if no bytes found see if MusicBrainz has cover art
        # if not ba:
        #     coverArtBytes = mb.lookupCoverArt(release.MusicBrainzId)
        #     if coverArtBytes:
        #         try:
        #             img = Image.open(io.BytesIO(coverArtBytes))
        #             img.thumbnail(self.thumbnailSize)
        #             b = io.BytesIO()
        #             img.save(b, "JPEG")
        #             ba = b.getvalue()
        #         except:
        #             pass
        if sr.genres:
            createDattabaseModelFromSearchModelRelease.genres = []
            for genre in sr.genres:
                dbGenre = self.session.query(Genre).filter(Genre.name == genre.name).first()
                if not dbGenre:
                    g = Genre()
                    g.name = genre.name
                    g.roadieId = genre.roadieId
                    createDattabaseModelFromSearchModelRelease.genres.append(g)
                else:
                    createDattabaseModelFromSearchModelRelease.genres.append(dbGenre)
        if sr.releaseLabels:
            createDattabaseModelFromSearchModelRelease.releaseLabels = []
            for srReleaseLabel in sr.releaseLabels:
                l = self._getLabelFromDatabase(srReleaseLabel.label.name)
                if not l:
                    l = Label()
                    l.roadieId = srReleaseLabel.label.roadieId
                    l.musicBrainzId = srReleaseLabel.label.musicBrainzId
                    l.beginDate = srReleaseLabel.label.beginDate
                    l.end = srReleaseLabel.label.endDate
                    l.imageUrl = srReleaseLabel.label.imageUrl
                    l.tags = srReleaseLabel.label.tags
                    if srReleaseLabel.label.alternateNames:
                        srLabelAlternateNames = []
                        for srLabelAn in srReleaseLabel.label.alternateNames:
                            srLabelAlternateNames.append(srLabelAn.replace("|", ","))
                        l.alternateNames = srLabelAlternateNames
                    l.sortName = srReleaseLabel.label.sortName
                    l.name = srReleaseLabel.label.name
                if l:
                    rl = ReleaseLabel()
                    rl.roadieId = srReleaseLabel.roadieId
                    rl.catalogNumber = srReleaseLabel.catalogNumber
                    rl.beginDate = parseDate(srReleaseLabel.beginDate)
                    rl.endDate = parseDate(srReleaseLabel.endDate)
                    rl.label = l
                    if rl not in createDattabaseModelFromSearchModelRelease.releaseLabels:
                        createDattabaseModelFromSearchModelRelease.releaseLabels.append(rl)
        if sr.media:
            createDattabaseModelFromSearchModelRelease.media = []
            for srMedia in sr.media:
                media = ReleaseMedia()
                media.roadieId = srMedia.roadieId
                media.releaseMediaNumber = int(srMedia.releaseMediaNumber)
                # The first media is release 1 not release 0
                if media.releaseMediaNumber < 1:
                    media.releaseMediaNumber = 1
                media.releaseSubTitle = srMedia.releaseSubTitle
                media.trackCount = srMedia.trackCount
                if srMedia.tracks:
                    media.tracks = []
                    for srTrack in srMedia.tracks:
                        track = Track()
                        track.roadieId = srTrack.roadieId
                        track.partTitles = srTrack.partTitles
                        track.musicBrainzId = srTrack.musicBrainzId
                        track.amgId = srTrack.amgId
                        track.spotifyId = srTrack.spotifyId
                        track.title = srTrack.title
                        track.trackNumber = srTrack.trackNumber
                        track.duration = srTrack.duration
                        track.tags = srTrack.tags
                        track.alternateNames = []
                        cleanedTitle = createCleanedName(srTrack.title)
                        if cleanedTitle != srTrack.title.lower().strip():
                            track.alternateNames.append(cleanedTitle)
                        media.tracks.append(track)
                createDattabaseModelFromSearchModelRelease.media.append(media)
            createDattabaseModelFromSearchModelRelease.mediaCount = len(
                createDattabaseModelFromSearchModelRelease.media)
        return createDattabaseModelFromSearchModelRelease
Esempio n. 2
0
    def scan(self, folder, artist, release):
        """
        Scan the given folder and update, insert or mark missing track information
        :param folder: str
        :param artist: Artist
        :param release: Release
        :return: int
        """
        if self.readOnly:
            self.logger.debug("[Read Only] Would Process Folder [" + folder + "] With Artist [" + str(artist) + "]")
            return None
        if not artist:
            self.logger.debug("! scanner.scan given Invalid Artist")
            raise RuntimeError("Invalid Artist")
        if not release:
            self.logger.debug("! scanner.scan given Invalid Release")
            raise RuntimeError("Invalid Release")
        if not folder:
            self.logger.debug("! scanner.scan given Invalid Folder")
            raise RuntimeError("Invalid Folder")
        folderHead, folderTail = os.path.split(folder)
        folderHeadNoLibrary = folderHead.replace(self.config['ROADIE_LIBRARY_FOLDER'], "")
        trackFilePath = os.path.join(folderHeadNoLibrary, folderTail)
        startTime = arrow.utcnow().datetime
        self.logger.info("-> Scanning Folder [" + folder + "] " +
                         "Artist Folder [" + self.artistFolder(artist) + "] ")
        # Get any existing tracks for folder and verify; update if ID3 tags are different
        if not self.readOnly:
            existingTracksChecked = 0
            for track in self.dbSession.query(Track).filter(Track.filePath == trackFilePath).all():
                existingTracksChecked += 1
                filename = self.pathToTrack(track)
                # File no longer exists for track
                if not os.path.isfile(filename):
                    if not self.readOnly:
                        self._markTrackMissing(track, filename)
                else:
                    id3 = ID3(filename)
                    # File has invalid ID3 tags now
                    if not id3.isValid():
                        self.logger.warn("! Track Has Invalid or Missing ID3 Tags [" + filename + "]")
                        if not self.readOnly:
                            try:
                                os.remove(filename)
                            except OSError:
                                pass
                            self._markTrackMissing(track, filename)
                    else:
                        try:
                            id3Hash = self.makeTrackHash(artist.roadieId, str(id3))
                            if id3Hash != track.Hash:
                                if not self.readOnly:
                                    self.logger.warn("x Hash Mismatch [" + track.Title + "]")
                                    self._markTrackMissing(track, filename)
                        except:
                            pass
            self.logger.debug(
                    "-- Checked [" + str(existingTracksChecked) + "] Existing Tracks for [" + str(trackFilePath) + "]")

        # For each file found in folder get ID3 info and insert record into Track DB
        foundReleaseTracks = 0
        createdReleaseTracks = 0
        scannedMp3Files = 0
        releaseMediaTrackCount = 0
        for mp3 in self.inboundMp3Files(folder):
            id3 = ID3(mp3)
            if id3 is not None:
                cleanedTitle = createCleanedName(id3.title)
                if not id3.isValid():
                    self.logger.warn("! Track Has Invalid or Missing ID3 Tags [" + mp3 + "]")
                else:
                    head, tail = os.path.split(mp3)
                    headNoLibrary = head.replace(self.config['ROADIE_LIBRARY_FOLDER'], "")
                    trackHash = self.makeTrackHash(artist.roadieId, str(id3))
                    track = None
                    mp3FileSize = os.path.getsize(mp3)
                    id3MediaNumber = id3.disc
                    # The first media is release 1 not release 0
                    if id3MediaNumber < 1:
                        id3MediaNumber = 1
                    releaseMedia = None
                    for releaseMediaFind in release.media:
                        if releaseMediaFind.releaseMediaNumber == id3MediaNumber:
                            releaseMedia = releaseMediaFind
                            for releaseTrack in releaseMediaFind.tracks:
                                if isEqual(releaseTrack.trackNumber, id3.track) or isEqual(releaseTrack.hash,
                                                                                           trackHash):
                                    track = releaseTrack
                                    releaseMediaTrackCount = releaseMediaFind.trackCount
                                    break
                                else:
                                    continue
                                break
                            else:
                                continue
                            break
                    if not track:
                        # If the track isn't found on the release media see if it exists on another and move it
                        existingTrackByHash = self.dbSession.query(Track).filter(Track.hash == trackHash).first()
                        if existingTrackByHash:
                            track = existingTrackByHash
                            if releaseMedia:
                                oldMediaId = track.releaseMediaId
                                track.releaseMediaId = releaseMedia.id
                                self.logger.warn("=> Moved Track Id [" + str(track.id) + "] " +
                                                 "to ReleaseMedia Id [" + str(releaseMedia.id) + "] " +
                                                 "was on ReleaseMedia Id [" + str(oldMediaId) + "]")
                    if not track:
                        createdReleaseTracks += 1
                        if not releaseMedia:
                            releaseMedia = ReleaseMedia()
                            releaseMedia.tracks = []
                            releaseMedia.status = 1
                            releaseMedia.trackCount = 1
                            releaseMedia.releaseMediaNumber = id3MediaNumber
                            releaseMedia.roadieId = str(uuid.uuid4())
                            if not release.media:
                                release.media = []
                            release.media.append(releaseMedia)
                            release.mediaCount = len(release.media)
                            self.logger.info("+ Added ReleaseMedia [" + str(releaseMedia.info()) + "] To Release")
                        track = Track()
                        track.fileName = tail
                        track.filePath = headNoLibrary
                        track.hash = trackHash
                        track.fileSize = mp3FileSize
                        track.createdDate = arrow.utcnow().datetime
                        track.roadieId = str(uuid.uuid4())
                        track.title = id3.title
                        track.trackNumber = id3.track
                        track.duration = int(id3.length) * 1000
                        track.status = 1
                        track.partTitles = []
                        if id3.hasTrackArtist():
                            shouldMakeArtistIfNotFound = not release.isCastRecording()
                            ta = id3.getTrackArtist()
                            trackArtist = self.artistFactory.get(ta, shouldMakeArtistIfNotFound)
                            if trackArtist:
                                track.artistId = trackArtist.id
                            elif not shouldMakeArtistIfNotFound:
                                track.partTitles.append(ta)
                                self.logger.info("+ Added Track PartTitle [" + str(ta) + "]")
                            if id3.artists:
                                ta = "/".join(id3.artists)
                                track.partTitles.append(ta)
                                self.logger.info("+ Added Track PartTitle [" + str(ta) + "]")
                        track.tags = []
                        track.alternateNames = []
                        if cleanedTitle != id3.title.lower().strip():
                            track.alternateNames.append(cleanedTitle)
                        releaseMedia.tracks.append(track)
                        releaseMedia.trackCount += 1
                        releaseMediaTrackCount = releaseMedia.trackCount
                        self.logger.info("+ Added Track [" + str(track.info()) + "] To ReleaseMedia")

                    elif not self.readOnly:
                        foundReleaseTracks += 1
                        try:
                            trackFullPath = self.pathToTrack(track)
                            isFilePathSame = os.path.samefile(trackFullPath, mp3)
                        except:
                            trackFullPath = None
                            isFilePathSame = False
                        isFileSizeSame = isEqual(track.fileSize, mp3FileSize)
                        isHashSame = isEqual(track.hash, trackHash)
                        if not isFilePathSame or not isFileSizeSame or not isHashSame:
                            track.fileName = tail
                            track.filePath = headNoLibrary
                            track.fileSize = mp3FileSize
                            track.hash = trackHash
                            track.lastUpdated = arrow.utcnow().datetime
                            release.lastUpdated = track.lastUpdated
                            if releaseMedia:
                                releaseMedia.lastUpdated = track.lastUpdated
                            if not track.alternateNames:
                                track.alternateNames = []
                            if cleanedTitle != track.title.lower().strip() and cleanedTitle not in track.alternateNames:
                                track.alternateNames.append(cleanedTitle)
                            self.logger.info("* Updated Track [" + str(track.info()) + "]: " +
                                             "isFilePathSame [" + str(
                                    isFilePathSame) + "] (" + str(trackFullPath) + ":" + str(mp3) + ") " +
                                             "isFileSizeSame [" + str(
                                    isFileSizeSame) + "] (" + str(track.fileSize) + ":" + str(mp3FileSize) + ") " +
                                             "isHashSame [" + str(
                                    isHashSame) + "] (" + str(track.hash) + ":" + str(trackHash) + ") ")
                    scannedMp3Files += 1

        elapsedTime = arrow.utcnow().datetime - startTime
        mp3FilesInFolder = self.mp3FileCountForFolder(folder)
        if mp3FilesInFolder == releaseMediaTrackCount:
            release.libraryStatus = 'Complete'
            if release.trackCount == 0:
                release.trackCount = mp3FilesInFolder
        elif not mp3FilesInFolder:
            release.libraryStatus = 'Missing'
        else:
            release.libraryStatus = 'Incomplete'

        self.logger.info("<- Scanning Folder [" + str(folder.encode('utf-8')) + "] " +
                         "Complete, Scanned [" + ('%02d' % scannedMp3Files) + "] " +
                         "Mp3 Files: Created [" + str(createdReleaseTracks) + "] Release Tracks, " +
                         "Found [" + str(foundReleaseTracks) + "] Release Tracks. " +
                         "Exist in Release Folder [" + str(mp3FilesInFolder) + "] " +
                         "Elapsed Time [" + str(elapsedTime) + "]")
        return scannedMp3Files