Пример #1
0
 def create(self, artist, title, trackCount, releaseDate):
     if not artist or not title or not trackCount or not releaseDate:
         return None
     release = Release()
     release.title = title
     release.releaseDate = parseDate(releaseDate)
     release.trackCount = trackCount
     release.artistId = artist.id
     release.createdDate = arrow.utcnow().datetime
     release.roadieId = str(uuid.uuid4())
     release.alternateNames = []
     cleanedTitle = createCleanedName(title)
     if cleanedTitle != title.lower().strip():
         release.alternateNames.append(cleanedTitle)
     return release
Пример #2
0
    def process(self, **kwargs):
        """
        Process folder using the passed folder

        """
        try:
            inboundFolder = kwargs.pop('folder', self.InboundFolder)
            forceFolderScan = kwargs.pop('forceFolderScan', False)
            isReleaseFolder = kwargs.pop('isReleaseFolder', False)
            doValidateArtist = kwargs.pop('doValidateArtist', True)
            self.logger.info("Processing Folder [" + inboundFolder + "] Flush [" + str(self.flushBefore) + "]")
            scanner = Scanner(self.config, self.conn, self.session, self.artistFactory, self.readOnly)
            startTime = arrow.utcnow().datetime
            newMp3Folder = None
            lastID3Artist = None
            lastID3Album = None
            artist = None
            release = None
            mp3FoldersProcessed = []
            artistsReleasesProcessed = defaultdict(list)
            validator = Validator(self.config, self.conn, self.session, self.readOnly)
            releaseFolder = None
            # Get all the folder in the InboundFolder
            for mp3Folder in ProcessorBase.allDirectoriesInDirectory(inboundFolder, isReleaseFolder):
                try:
                    try:
                        mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(mp3Folder))
                    except:
                        mp3FolderMtime = None
                        pass
                    if mp3FolderMtime and not self._doProcessFolder(mp3Folder, mp3FolderMtime, forceFolderScan):
                        self.logger.info("Skipping Folder [" + mp3Folder + "] No Changes Detected")
                        continue

                    foundMp3Files = 0

                    # Do any conversions
                    if not self.readOnly:
                        Convertor(mp3Folder)

                    # Delete any empty folder if enabled
                    try:
                        if not os.listdir(mp3Folder) and not self.dontDeleteInboundFolders:
                            try:
                                self.logger.warn("X Deleted Empty Folder [" + mp3Folder + "]")
                                if not self.readOnly:
                                    os.rmdir(mp3Folder)
                            except OSError:
                                self.logger.error("Error Deleting [" + mp3Folder + "]")
                            continue
                    except:
                        pass

                    # Get all the MP3 files in the Folder and process
                    for rootFolder, mp3 in ProcessorBase.folderMp3Files(mp3Folder):
                        printableMp3 = mp3.encode('ascii', 'ignore').decode('utf-8')
                        self.logger.debug("Processing MP3 File [" + printableMp3 + "]")
                        id3StartTime = arrow.utcnow().datetime
                        id3 = ID3(mp3, self.processingOptions)
                        id3ElapsedTime = arrow.utcnow().datetime - id3StartTime
                        if id3 is not None:
                            if not id3.isValid():
                                self.logger.warn("! Track Has Invalid or Missing ID3 Tags [" + printableMp3 + "]")
                            else:
                                foundMp3Files += 1
                                # Get Artist
                                if lastID3Artist != id3.getReleaseArtist():
                                    artist = None
                                if not artist:
                                    lastID3Artist = id3.getReleaseArtist()
                                    r = release
                                    if not r:
                                        r = Release()
                                        r.title = id3.album
                                        r.artist = Artist()
                                        r.artist.name = id3.getReleaseArtist()
                                    artist = self.artistFactory.get(id3.getReleaseArtist(), not r.isCastRecording())
                                if artist and artist.isLocked:
                                    self.logger.debug(
                                        "Skipping Processing Track [" + printableMp3 + "], Artist [" + str(
                                            artist) + "] Is Locked")
                                    continue
                                if self.flushBefore:
                                    if artist.isLocked:
                                        self.logger.debug(
                                            "Skipping Flushing Artist [" + printableMp3 + "], Artist [" + str(
                                                artist) + "] Is Locked")
                                        continue
                                    else:
                                        for release in artist.releases:
                                            release.genres = []
                                            self.session.delete(release)
                                        self.session.commit()
                                if not artist:
                                    self.logger.warn(
                                        "! Unable to Find Artist [" + id3.getReleaseArtist() + "] for Mp3 [" + printableMp3 + "]")
                                    continue
                                # Get the Release
                                if lastID3Album != id3.album:
                                    release = None
                                if not release:
                                    lastID3Album = id3.album
                                    release = self.releaseFactory.get(artist, id3.album)
                                    if release:
                                        # Was found now see if needs update based on id3 tag info
                                        id3ReleaseDate = parseDate(id3.year)
                                        if not release.releaseDate == id3ReleaseDate and id3ReleaseDate:
                                            release.releaseDate = id3ReleaseDate
                                        if id3.imageBytes and not release.thumbnail:
                                            try:
                                                img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB')
                                                img.thumbnail(self.thumbnailSize)
                                                b = io.BytesIO()
                                                img.save(b, "JPEG")
                                                release.thumbnail = b.getvalue()
                                            except:
                                                pass

                                    else:
                                        # Was not found in any Searcher create and add
                                        self.logger.debug("Release [" + id3.album + "] Not Found By Factory")
                                        release = self.releaseFactory.create(artist,
                                                                             string.capwords(id3.album),
                                                                             1,
                                                                             id3.year)
                                        if not release:
                                            self.logger.warn("! Unable to Create Album [" + id3.album +
                                                             "] For Track [" + printableMp3 + "]")
                                            continue
                                        if release:
                                            if id3.imageBytes:
                                                try:
                                                    img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB')
                                                    img.thumbnail(self.thumbnailSize)
                                                    b = io.BytesIO()
                                                    img.save(b, "JPEG")
                                                    release.thumbnail = b.getvalue()
                                                except:
                                                    pass
                                            release.status = 1
                                            self.releaseFactory.add(release)
                                            self.logger.info(
                                                "+ Processor Added Release [" + str(release.info()) + "]")
                                            self.session.commit()
                                if self.shouldMoveToLibrary(artist, id3, mp3):
                                    newMp3 = self.moveToLibrary(artist, id3, mp3)
                                    head, tail = os.path.split(newMp3)
                                    newMp3Folder = head
                    if artist and release:
                        if not release.releaseDate and release.media:
                            for media in release.media:
                                if media.tracks:
                                    for track in media.tracks:
                                        if track.filePath:
                                            release.releaseDate = parseDate(track.filePath.split('\\')[1][1:5])
                                            break
                                        else:
                                            continue
                                        break
                                    else:
                                        continue
                                    break
                            else:
                                continue
                            break
                        if not release.releaseDate:
                            release.releaseDate = parseDate(id3.year)
                        releaseFolder = self.albumFolder(artist, release.releaseDate.strftime('%Y'), release.title)
                        if newMp3Folder and newMp3Folder not in mp3FoldersProcessed:
                            for coverImage in self.releaseCoverImages(mp3Folder):
                                try:
                                    im = Image.open(coverImage).convert('RGB')
                                    newPath = os.path.join(newMp3Folder, "cover.jpg")
                                    if (not os.path.isfile(newPath) or not os.path.samefile(coverImage,
                                                                                            newPath)) and not self.readOnly:
                                        im.save(newPath)
                                        self.logger.info(
                                            "+ Copied Cover File [" + coverImage + "] => [" + newPath + "]")
                                except:
                                    self.logger.exception("Error Copying File [" + coverImage + "]")
                                    pass
                            mp3FoldersProcessed.append(newMp3Folder)
                        if release not in artistsReleasesProcessed[artist]:
                            artistsReleasesProcessed[artist].append(release)
                    if not self.readOnly and artist and release:
                        if self.shouldDeleteFolder(mp3Folder):
                            try:
                                shutil.rmtree(mp3Folder)
                                self.logger.debug("x Deleted Processed Folder [" + mp3Folder + "]")
                            except OSError:
                                self.logger.warn("Could Not Delete Folder [" + mp3Folder + "]")
                                pass

                    self.session.commit()
                    gc.collect()

                except:
                    self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions")
                    self.session.rollback()
                if releaseFolder:
                    scanner.scan(releaseFolder, artist, release)
                    # Sync the counts as some release media and release tracks where added by the processor
                    release.mediaCount = len(release.media)
                    release.trackCount = 0
                    for media in release.media:
                        media.trackCount = len(media.tracks)
                        release.trackCount += len(media.tracks)
                    releaseFolder = None
            self.session.commit()
            if artistsReleasesProcessed and doValidateArtist:
                self.logger.info("Validating [" + str(len(artistsReleasesProcessed)) + "] Artists")
                for artistToValidate, artistReleasesToValidate in artistsReleasesProcessed.items():
                    artistFolder = self.artistFolder(artistToValidate)
                    try:
                        mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(artistFolder))
                    except:
                        mp3FolderMtime = None
                        pass
                    if mp3FolderMtime and not self._doProcessFolder(artistFolder, mp3FolderMtime, forceFolderScan):
                        self.logger.info("== Skipping Artist Folder [" + artistFolder + "] No Changes Detected")
                        continue
                    for artistReleaseToValidate in artistReleasesToValidate:
                        validator.validate(artistToValidate, artistReleaseToValidate)
            elapsedTime = arrow.utcnow().datetime - startTime
            self.logger.info("Processing Complete. Elapsed Time [" + str(elapsedTime) + "]")
        except:
            self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions")
            self.session.rollback()
Пример #3
0
    def _createDatabaseModelFromSearchModel(self, artist, title, sr):
        """
        Take the given SearchResult Release Model and create a Database Model
        :type artist: Artist
        :type title: str
        :type sr: searchEngines.models.Release.Release
        """
        createDattabaseModelFromSearchModelRelease = Release()
        printableTitle = title.encode('ascii', 'ignore').decode('utf-8')
        releaseByExternalIds = self._getFromDatabaseByExternalIds(sr.musicBrainzId,
                                                                  sr.iTunesId,
                                                                  sr.lastFMId,
                                                                  sr.amgId,
                                                                  sr.spotifyId)
        if releaseByExternalIds:
            if not releaseByExternalIds.alternateNames:
                releaseByExternalIds.alternateNames = []
            if title not in releaseByExternalIds.alternateNames:
                self.logger.debug("Found Title By External Ids [" +
                                  releaseByExternalIds.title.encode('ascii', 'ignore')
                                  .decode('utf-8') + "] Added [" +
                                  printableTitle + "] To AlternateNames")
                if not releaseByExternalIds.alternateNames:
                    releaseByExternalIds.alternateNames = []
                releaseByExternalIds.alternateNames.append(title)
                releaseByExternalIds.lastUpdated = arrow.utcnow().datetime
                self.session.commit()
            return releaseByExternalIds
        createDattabaseModelFromSearchModelRelease.artist = artist
        createDattabaseModelFromSearchModelRelease.roadieId = sr.roadieId
        createDattabaseModelFromSearchModelRelease.title = title
        createDattabaseModelFromSearchModelRelease.releaseDate = parseDate(sr.releaseDate)
        createDattabaseModelFromSearchModelRelease.trackCount = sr.trackCount
        createDattabaseModelFromSearchModelRelease.mediaCount = sr.mediaCount
        createDattabaseModelFromSearchModelRelease.thumbnail = sr.thumbnail
        createDattabaseModelFromSearchModelRelease.profile = sr.profile
        if sr.releaseType == SearchReleaseType.Album:
            createDattabaseModelFromSearchModelRelease.releaseType = 'Album'
        elif sr.releaseType == SearchReleaseType.EP:
            createDattabaseModelFromSearchModelRelease.releaseType = 'EP'
        elif sr.releaseType == SearchReleaseType.Single:
            createDattabaseModelFromSearchModelRelease.releaseType = 'Single'
        createDattabaseModelFromSearchModelRelease.iTunesId = sr.iTunesId
        createDattabaseModelFromSearchModelRelease.amgId = sr.amgId
        createDattabaseModelFromSearchModelRelease.lastFMId = sr.lastFMId
        createDattabaseModelFromSearchModelRelease.lastFMSummary = sr.lastFMSummary
        createDattabaseModelFromSearchModelRelease.musicBrainzId = sr.musicBrainzId
        createDattabaseModelFromSearchModelRelease.spotifyId = sr.spotifyId
        createDattabaseModelFromSearchModelRelease.amgId = sr.amgId
        createDattabaseModelFromSearchModelRelease.tags = sr.tags
        createDattabaseModelFromSearchModelRelease.alternateNames = sr.alternateNames

        createDattabaseModelFromSearchModelRelease.urls = sr.urls
        if sr.images:
            createDattabaseModelFromSearchModelReleaseimages = []
            for image in sr.images:
                if image.image:
                    i = Image()
                    i.roadieId = image.roadieId
                    i.url = image.url
                    i.caption = image.caption
                    i.image = image.image
                    i.signature = image.signature
                    createDattabaseModelFromSearchModelReleaseimages.append(i)
            createDattabaseModelFromSearchModelRelease.images = createDattabaseModelFromSearchModelReleaseimages
            self.logger.debug(
                "= Added [" + str(len(createDattabaseModelFromSearchModelRelease.images)) + "] Images to Release")

        # TODO
        # See if cover file found in Release Folder
        # coverFile = os.path.join(mp3Folder, "cover.jpg")
        # if os.path.isfile(coverFile):
        #     ba = self.readImageThumbnailBytesFromFile(coverFile)
        # else:
        #     coverFile = os.path.join(mp3Folder, "front.jpg")
        #     if os.path.isfile(coverFile):
        #         ba = self.readImageThumbnailBytesFromFile(coverFile)
        # # if no bytes found see if MusicBrainz has cover art
        # if not ba:
        #     coverArtBytes = mb.lookupCoverArt(release.MusicBrainzId)
        #     if coverArtBytes:
        #         try:
        #             img = Image.open(io.BytesIO(coverArtBytes))
        #             img.thumbnail(self.thumbnailSize)
        #             b = io.BytesIO()
        #             img.save(b, "JPEG")
        #             ba = b.getvalue()
        #         except:
        #             pass
        if sr.genres:
            createDattabaseModelFromSearchModelRelease.genres = []
            for genre in sr.genres:
                dbGenre = self.session.query(Genre).filter(Genre.name == genre.name).first()
                if not dbGenre:
                    g = Genre()
                    g.name = genre.name
                    g.roadieId = genre.roadieId
                    createDattabaseModelFromSearchModelRelease.genres.append(g)
                else:
                    createDattabaseModelFromSearchModelRelease.genres.append(dbGenre)
        if sr.releaseLabels:
            createDattabaseModelFromSearchModelRelease.releaseLabels = []
            for srReleaseLabel in sr.releaseLabels:
                l = self._getLabelFromDatabase(srReleaseLabel.label.name)
                if not l:
                    l = Label()
                    l.roadieId = srReleaseLabel.label.roadieId
                    l.musicBrainzId = srReleaseLabel.label.musicBrainzId
                    l.beginDate = srReleaseLabel.label.beginDate
                    l.end = srReleaseLabel.label.endDate
                    l.imageUrl = srReleaseLabel.label.imageUrl
                    l.tags = srReleaseLabel.label.tags
                    if srReleaseLabel.label.alternateNames:
                        srLabelAlternateNames = []
                        for srLabelAn in srReleaseLabel.label.alternateNames:
                            srLabelAlternateNames.append(srLabelAn.replace("|", ","))
                        l.alternateNames = srLabelAlternateNames
                    l.sortName = srReleaseLabel.label.sortName
                    l.name = srReleaseLabel.label.name
                if l:
                    rl = ReleaseLabel()
                    rl.roadieId = srReleaseLabel.roadieId
                    rl.catalogNumber = srReleaseLabel.catalogNumber
                    rl.beginDate = parseDate(srReleaseLabel.beginDate)
                    rl.endDate = parseDate(srReleaseLabel.endDate)
                    rl.label = l
                    if rl not in createDattabaseModelFromSearchModelRelease.releaseLabels:
                        createDattabaseModelFromSearchModelRelease.releaseLabels.append(rl)
        if sr.media:
            createDattabaseModelFromSearchModelRelease.media = []
            for srMedia in sr.media:
                media = ReleaseMedia()
                media.roadieId = srMedia.roadieId
                media.releaseMediaNumber = int(srMedia.releaseMediaNumber)
                # The first media is release 1 not release 0
                if media.releaseMediaNumber < 1:
                    media.releaseMediaNumber = 1
                media.releaseSubTitle = srMedia.releaseSubTitle
                media.trackCount = srMedia.trackCount
                if srMedia.tracks:
                    media.tracks = []
                    for srTrack in srMedia.tracks:
                        track = Track()
                        track.roadieId = srTrack.roadieId
                        track.partTitles = srTrack.partTitles
                        track.musicBrainzId = srTrack.musicBrainzId
                        track.amgId = srTrack.amgId
                        track.spotifyId = srTrack.spotifyId
                        track.title = srTrack.title
                        track.trackNumber = srTrack.trackNumber
                        track.duration = srTrack.duration
                        track.tags = srTrack.tags
                        track.alternateNames = []
                        cleanedTitle = createCleanedName(srTrack.title)
                        if cleanedTitle != srTrack.title.lower().strip():
                            track.alternateNames.append(cleanedTitle)
                        media.tracks.append(track)
                createDattabaseModelFromSearchModelRelease.media.append(media)
            createDattabaseModelFromSearchModelRelease.mediaCount = len(
                createDattabaseModelFromSearchModelRelease.media)
        return createDattabaseModelFromSearchModelRelease