def process(self, **kwargs): """ Process folder using the passed folder """ try: inboundFolder = kwargs.pop('folder', self.InboundFolder) forceFolderScan = kwargs.pop('forceFolderScan', False) isReleaseFolder = kwargs.pop('isReleaseFolder', False) doValidateArtist = kwargs.pop('doValidateArtist', True) self.logger.info("Processing Folder [" + inboundFolder + "] Flush [" + str(self.flushBefore) + "]") scanner = Scanner(self.config, self.conn, self.session, self.artistFactory, self.readOnly) startTime = arrow.utcnow().datetime newMp3Folder = None lastID3Artist = None lastID3Album = None artist = None release = None mp3FoldersProcessed = [] artistsReleasesProcessed = defaultdict(list) validator = Validator(self.config, self.conn, self.session, self.readOnly) releaseFolder = None # Get all the folder in the InboundFolder for mp3Folder in ProcessorBase.allDirectoriesInDirectory(inboundFolder, isReleaseFolder): try: try: mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(mp3Folder)) except: mp3FolderMtime = None pass if mp3FolderMtime and not self._doProcessFolder(mp3Folder, mp3FolderMtime, forceFolderScan): self.logger.info("Skipping Folder [" + mp3Folder + "] No Changes Detected") continue foundMp3Files = 0 # Do any conversions if not self.readOnly: Convertor(mp3Folder) # Delete any empty folder if enabled try: if not os.listdir(mp3Folder) and not self.dontDeleteInboundFolders: try: self.logger.warn("X Deleted Empty Folder [" + mp3Folder + "]") if not self.readOnly: os.rmdir(mp3Folder) except OSError: self.logger.error("Error Deleting [" + mp3Folder + "]") continue except: pass # Get all the MP3 files in the Folder and process for rootFolder, mp3 in ProcessorBase.folderMp3Files(mp3Folder): printableMp3 = mp3.encode('ascii', 'ignore').decode('utf-8') self.logger.debug("Processing MP3 File [" + printableMp3 + "]") id3StartTime = arrow.utcnow().datetime id3 = ID3(mp3, self.processingOptions) id3ElapsedTime = arrow.utcnow().datetime - id3StartTime if id3 is not None: if not id3.isValid(): self.logger.warn("! Track Has Invalid or Missing ID3 Tags [" + printableMp3 + "]") else: foundMp3Files += 1 # Get Artist if lastID3Artist != id3.getReleaseArtist(): artist = None if not artist: lastID3Artist = id3.getReleaseArtist() r = release if not r: r = Release() r.title = id3.album r.artist = Artist() r.artist.name = id3.getReleaseArtist() artist = self.artistFactory.get(id3.getReleaseArtist(), not r.isCastRecording()) if artist and artist.isLocked: self.logger.debug( "Skipping Processing Track [" + printableMp3 + "], Artist [" + str( artist) + "] Is Locked") continue if self.flushBefore: if artist.isLocked: self.logger.debug( "Skipping Flushing Artist [" + printableMp3 + "], Artist [" + str( artist) + "] Is Locked") continue else: for release in artist.releases: release.genres = [] self.session.delete(release) self.session.commit() if not artist: self.logger.warn( "! Unable to Find Artist [" + id3.getReleaseArtist() + "] for Mp3 [" + printableMp3 + "]") continue # Get the Release if lastID3Album != id3.album: release = None if not release: lastID3Album = id3.album release = self.releaseFactory.get(artist, id3.album) if release: # Was found now see if needs update based on id3 tag info id3ReleaseDate = parseDate(id3.year) if not release.releaseDate == id3ReleaseDate and id3ReleaseDate: release.releaseDate = id3ReleaseDate if id3.imageBytes and not release.thumbnail: try: img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB') img.thumbnail(self.thumbnailSize) b = io.BytesIO() img.save(b, "JPEG") release.thumbnail = b.getvalue() except: pass else: # Was not found in any Searcher create and add self.logger.debug("Release [" + id3.album + "] Not Found By Factory") release = self.releaseFactory.create(artist, string.capwords(id3.album), 1, id3.year) if not release: self.logger.warn("! Unable to Create Album [" + id3.album + "] For Track [" + printableMp3 + "]") continue if release: if id3.imageBytes: try: img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB') img.thumbnail(self.thumbnailSize) b = io.BytesIO() img.save(b, "JPEG") release.thumbnail = b.getvalue() except: pass release.status = 1 self.releaseFactory.add(release) self.logger.info( "+ Processor Added Release [" + str(release.info()) + "]") self.session.commit() if self.shouldMoveToLibrary(artist, id3, mp3): newMp3 = self.moveToLibrary(artist, id3, mp3) head, tail = os.path.split(newMp3) newMp3Folder = head if artist and release: if not release.releaseDate and release.media: for media in release.media: if media.tracks: for track in media.tracks: if track.filePath: release.releaseDate = parseDate(track.filePath.split('\\')[1][1:5]) break else: continue break else: continue break else: continue break if not release.releaseDate: release.releaseDate = parseDate(id3.year) releaseFolder = self.albumFolder(artist, release.releaseDate.strftime('%Y'), release.title) if newMp3Folder and newMp3Folder not in mp3FoldersProcessed: for coverImage in self.releaseCoverImages(mp3Folder): try: im = Image.open(coverImage).convert('RGB') newPath = os.path.join(newMp3Folder, "cover.jpg") if (not os.path.isfile(newPath) or not os.path.samefile(coverImage, newPath)) and not self.readOnly: im.save(newPath) self.logger.info( "+ Copied Cover File [" + coverImage + "] => [" + newPath + "]") except: self.logger.exception("Error Copying File [" + coverImage + "]") pass mp3FoldersProcessed.append(newMp3Folder) if release not in artistsReleasesProcessed[artist]: artistsReleasesProcessed[artist].append(release) if not self.readOnly and artist and release: if self.shouldDeleteFolder(mp3Folder): try: shutil.rmtree(mp3Folder) self.logger.debug("x Deleted Processed Folder [" + mp3Folder + "]") except OSError: self.logger.warn("Could Not Delete Folder [" + mp3Folder + "]") pass self.session.commit() gc.collect() except: self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions") self.session.rollback() if releaseFolder: scanner.scan(releaseFolder, artist, release) # Sync the counts as some release media and release tracks where added by the processor release.mediaCount = len(release.media) release.trackCount = 0 for media in release.media: media.trackCount = len(media.tracks) release.trackCount += len(media.tracks) releaseFolder = None self.session.commit() if artistsReleasesProcessed and doValidateArtist: self.logger.info("Validating [" + str(len(artistsReleasesProcessed)) + "] Artists") for artistToValidate, artistReleasesToValidate in artistsReleasesProcessed.items(): artistFolder = self.artistFolder(artistToValidate) try: mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(artistFolder)) except: mp3FolderMtime = None pass if mp3FolderMtime and not self._doProcessFolder(artistFolder, mp3FolderMtime, forceFolderScan): self.logger.info("== Skipping Artist Folder [" + artistFolder + "] No Changes Detected") continue for artistReleaseToValidate in artistReleasesToValidate: validator.validate(artistToValidate, artistReleaseToValidate) elapsedTime = arrow.utcnow().datetime - startTime self.logger.info("Processing Complete. Elapsed Time [" + str(elapsedTime) + "]") except: self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions") self.session.rollback()
def _createDatabaseModelFromSearchModel(self, artist, title, sr): """ Take the given SearchResult Release Model and create a Database Model :type artist: Artist :type title: str :type sr: searchEngines.models.Release.Release """ createDattabaseModelFromSearchModelRelease = Release() printableTitle = title.encode('ascii', 'ignore').decode('utf-8') releaseByExternalIds = self._getFromDatabaseByExternalIds(sr.musicBrainzId, sr.iTunesId, sr.lastFMId, sr.amgId, sr.spotifyId) if releaseByExternalIds: if not releaseByExternalIds.alternateNames: releaseByExternalIds.alternateNames = [] if title not in releaseByExternalIds.alternateNames: self.logger.debug("Found Title By External Ids [" + releaseByExternalIds.title.encode('ascii', 'ignore') .decode('utf-8') + "] Added [" + printableTitle + "] To AlternateNames") if not releaseByExternalIds.alternateNames: releaseByExternalIds.alternateNames = [] releaseByExternalIds.alternateNames.append(title) releaseByExternalIds.lastUpdated = arrow.utcnow().datetime self.session.commit() return releaseByExternalIds createDattabaseModelFromSearchModelRelease.artist = artist createDattabaseModelFromSearchModelRelease.roadieId = sr.roadieId createDattabaseModelFromSearchModelRelease.title = title createDattabaseModelFromSearchModelRelease.releaseDate = parseDate(sr.releaseDate) createDattabaseModelFromSearchModelRelease.trackCount = sr.trackCount createDattabaseModelFromSearchModelRelease.mediaCount = sr.mediaCount createDattabaseModelFromSearchModelRelease.thumbnail = sr.thumbnail createDattabaseModelFromSearchModelRelease.profile = sr.profile if sr.releaseType == SearchReleaseType.Album: createDattabaseModelFromSearchModelRelease.releaseType = 'Album' elif sr.releaseType == SearchReleaseType.EP: createDattabaseModelFromSearchModelRelease.releaseType = 'EP' elif sr.releaseType == SearchReleaseType.Single: createDattabaseModelFromSearchModelRelease.releaseType = 'Single' createDattabaseModelFromSearchModelRelease.iTunesId = sr.iTunesId createDattabaseModelFromSearchModelRelease.amgId = sr.amgId createDattabaseModelFromSearchModelRelease.lastFMId = sr.lastFMId createDattabaseModelFromSearchModelRelease.lastFMSummary = sr.lastFMSummary createDattabaseModelFromSearchModelRelease.musicBrainzId = sr.musicBrainzId createDattabaseModelFromSearchModelRelease.spotifyId = sr.spotifyId createDattabaseModelFromSearchModelRelease.amgId = sr.amgId createDattabaseModelFromSearchModelRelease.tags = sr.tags createDattabaseModelFromSearchModelRelease.alternateNames = sr.alternateNames createDattabaseModelFromSearchModelRelease.urls = sr.urls if sr.images: createDattabaseModelFromSearchModelReleaseimages = [] for image in sr.images: if image.image: i = Image() i.roadieId = image.roadieId i.url = image.url i.caption = image.caption i.image = image.image i.signature = image.signature createDattabaseModelFromSearchModelReleaseimages.append(i) createDattabaseModelFromSearchModelRelease.images = createDattabaseModelFromSearchModelReleaseimages self.logger.debug( "= Added [" + str(len(createDattabaseModelFromSearchModelRelease.images)) + "] Images to Release") # TODO # See if cover file found in Release Folder # coverFile = os.path.join(mp3Folder, "cover.jpg") # if os.path.isfile(coverFile): # ba = self.readImageThumbnailBytesFromFile(coverFile) # else: # coverFile = os.path.join(mp3Folder, "front.jpg") # if os.path.isfile(coverFile): # ba = self.readImageThumbnailBytesFromFile(coverFile) # # if no bytes found see if MusicBrainz has cover art # if not ba: # coverArtBytes = mb.lookupCoverArt(release.MusicBrainzId) # if coverArtBytes: # try: # img = Image.open(io.BytesIO(coverArtBytes)) # img.thumbnail(self.thumbnailSize) # b = io.BytesIO() # img.save(b, "JPEG") # ba = b.getvalue() # except: # pass if sr.genres: createDattabaseModelFromSearchModelRelease.genres = [] for genre in sr.genres: dbGenre = self.session.query(Genre).filter(Genre.name == genre.name).first() if not dbGenre: g = Genre() g.name = genre.name g.roadieId = genre.roadieId createDattabaseModelFromSearchModelRelease.genres.append(g) else: createDattabaseModelFromSearchModelRelease.genres.append(dbGenre) if sr.releaseLabels: createDattabaseModelFromSearchModelRelease.releaseLabels = [] for srReleaseLabel in sr.releaseLabels: l = self._getLabelFromDatabase(srReleaseLabel.label.name) if not l: l = Label() l.roadieId = srReleaseLabel.label.roadieId l.musicBrainzId = srReleaseLabel.label.musicBrainzId l.beginDate = srReleaseLabel.label.beginDate l.end = srReleaseLabel.label.endDate l.imageUrl = srReleaseLabel.label.imageUrl l.tags = srReleaseLabel.label.tags if srReleaseLabel.label.alternateNames: srLabelAlternateNames = [] for srLabelAn in srReleaseLabel.label.alternateNames: srLabelAlternateNames.append(srLabelAn.replace("|", ",")) l.alternateNames = srLabelAlternateNames l.sortName = srReleaseLabel.label.sortName l.name = srReleaseLabel.label.name if l: rl = ReleaseLabel() rl.roadieId = srReleaseLabel.roadieId rl.catalogNumber = srReleaseLabel.catalogNumber rl.beginDate = parseDate(srReleaseLabel.beginDate) rl.endDate = parseDate(srReleaseLabel.endDate) rl.label = l if rl not in createDattabaseModelFromSearchModelRelease.releaseLabels: createDattabaseModelFromSearchModelRelease.releaseLabels.append(rl) if sr.media: createDattabaseModelFromSearchModelRelease.media = [] for srMedia in sr.media: media = ReleaseMedia() media.roadieId = srMedia.roadieId media.releaseMediaNumber = int(srMedia.releaseMediaNumber) # The first media is release 1 not release 0 if media.releaseMediaNumber < 1: media.releaseMediaNumber = 1 media.releaseSubTitle = srMedia.releaseSubTitle media.trackCount = srMedia.trackCount if srMedia.tracks: media.tracks = [] for srTrack in srMedia.tracks: track = Track() track.roadieId = srTrack.roadieId track.partTitles = srTrack.partTitles track.musicBrainzId = srTrack.musicBrainzId track.amgId = srTrack.amgId track.spotifyId = srTrack.spotifyId track.title = srTrack.title track.trackNumber = srTrack.trackNumber track.duration = srTrack.duration track.tags = srTrack.tags track.alternateNames = [] cleanedTitle = createCleanedName(srTrack.title) if cleanedTitle != srTrack.title.lower().strip(): track.alternateNames.append(cleanedTitle) media.tracks.append(track) createDattabaseModelFromSearchModelRelease.media.append(media) createDattabaseModelFromSearchModelRelease.mediaCount = len( createDattabaseModelFromSearchModelRelease.media) return createDattabaseModelFromSearchModelRelease