def process(self, **kwargs): """ Process folder using the passed folder """ try: inboundFolder = kwargs.pop('folder', self.InboundFolder) forceFolderScan = kwargs.pop('forceFolderScan', False) isReleaseFolder = kwargs.pop('isReleaseFolder', False) doValidateArtist = kwargs.pop('doValidateArtist', True) self.logger.info("Processing Folder [" + inboundFolder + "] Flush [" + str(self.flushBefore) + "]") scanner = Scanner(self.config, self.conn, self.session, self.artistFactory, self.readOnly) startTime = arrow.utcnow().datetime newMp3Folder = None lastID3Artist = None lastID3Album = None artist = None release = None mp3FoldersProcessed = [] artistsReleasesProcessed = defaultdict(list) validator = Validator(self.config, self.conn, self.session, self.readOnly) releaseFolder = None # Get all the folder in the InboundFolder for mp3Folder in ProcessorBase.allDirectoriesInDirectory(inboundFolder, isReleaseFolder): try: try: mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(mp3Folder)) except: mp3FolderMtime = None pass if mp3FolderMtime and not self._doProcessFolder(mp3Folder, mp3FolderMtime, forceFolderScan): self.logger.info("Skipping Folder [" + mp3Folder + "] No Changes Detected") continue foundMp3Files = 0 # Do any conversions if not self.readOnly: Convertor(mp3Folder) # Delete any empty folder if enabled try: if not os.listdir(mp3Folder) and not self.dontDeleteInboundFolders: try: self.logger.warn("X Deleted Empty Folder [" + mp3Folder + "]") if not self.readOnly: os.rmdir(mp3Folder) except OSError: self.logger.error("Error Deleting [" + mp3Folder + "]") continue except: pass # Get all the MP3 files in the Folder and process for rootFolder, mp3 in ProcessorBase.folderMp3Files(mp3Folder): printableMp3 = mp3.encode('ascii', 'ignore').decode('utf-8') self.logger.debug("Processing MP3 File [" + printableMp3 + "]") id3StartTime = arrow.utcnow().datetime id3 = ID3(mp3, self.processingOptions) id3ElapsedTime = arrow.utcnow().datetime - id3StartTime if id3 is not None: if not id3.isValid(): self.logger.warn("! Track Has Invalid or Missing ID3 Tags [" + printableMp3 + "]") else: foundMp3Files += 1 # Get Artist if lastID3Artist != id3.getReleaseArtist(): artist = None if not artist: lastID3Artist = id3.getReleaseArtist() r = release if not r: r = Release() r.title = id3.album r.artist = Artist() r.artist.name = id3.getReleaseArtist() artist = self.artistFactory.get(id3.getReleaseArtist(), not r.isCastRecording()) if artist and artist.isLocked: self.logger.debug( "Skipping Processing Track [" + printableMp3 + "], Artist [" + str( artist) + "] Is Locked") continue if self.flushBefore: if artist.isLocked: self.logger.debug( "Skipping Flushing Artist [" + printableMp3 + "], Artist [" + str( artist) + "] Is Locked") continue else: for release in artist.releases: release.genres = [] self.session.delete(release) self.session.commit() if not artist: self.logger.warn( "! Unable to Find Artist [" + id3.getReleaseArtist() + "] for Mp3 [" + printableMp3 + "]") continue # Get the Release if lastID3Album != id3.album: release = None if not release: lastID3Album = id3.album release = self.releaseFactory.get(artist, id3.album) if release: # Was found now see if needs update based on id3 tag info id3ReleaseDate = parseDate(id3.year) if not release.releaseDate == id3ReleaseDate and id3ReleaseDate: release.releaseDate = id3ReleaseDate if id3.imageBytes and not release.thumbnail: try: img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB') img.thumbnail(self.thumbnailSize) b = io.BytesIO() img.save(b, "JPEG") release.thumbnail = b.getvalue() except: pass else: # Was not found in any Searcher create and add self.logger.debug("Release [" + id3.album + "] Not Found By Factory") release = self.releaseFactory.create(artist, string.capwords(id3.album), 1, id3.year) if not release: self.logger.warn("! Unable to Create Album [" + id3.album + "] For Track [" + printableMp3 + "]") continue if release: if id3.imageBytes: try: img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB') img.thumbnail(self.thumbnailSize) b = io.BytesIO() img.save(b, "JPEG") release.thumbnail = b.getvalue() except: pass release.status = 1 self.releaseFactory.add(release) self.logger.info( "+ Processor Added Release [" + str(release.info()) + "]") self.session.commit() if self.shouldMoveToLibrary(artist, id3, mp3): newMp3 = self.moveToLibrary(artist, id3, mp3) head, tail = os.path.split(newMp3) newMp3Folder = head if artist and release: if not release.releaseDate and release.media: for media in release.media: if media.tracks: for track in media.tracks: if track.filePath: release.releaseDate = parseDate(track.filePath.split('\\')[1][1:5]) break else: continue break else: continue break else: continue break if not release.releaseDate: release.releaseDate = parseDate(id3.year) releaseFolder = self.albumFolder(artist, release.releaseDate.strftime('%Y'), release.title) if newMp3Folder and newMp3Folder not in mp3FoldersProcessed: for coverImage in self.releaseCoverImages(mp3Folder): try: im = Image.open(coverImage).convert('RGB') newPath = os.path.join(newMp3Folder, "cover.jpg") if (not os.path.isfile(newPath) or not os.path.samefile(coverImage, newPath)) and not self.readOnly: im.save(newPath) self.logger.info( "+ Copied Cover File [" + coverImage + "] => [" + newPath + "]") except: self.logger.exception("Error Copying File [" + coverImage + "]") pass mp3FoldersProcessed.append(newMp3Folder) if release not in artistsReleasesProcessed[artist]: artistsReleasesProcessed[artist].append(release) if not self.readOnly and artist and release: if self.shouldDeleteFolder(mp3Folder): try: shutil.rmtree(mp3Folder) self.logger.debug("x Deleted Processed Folder [" + mp3Folder + "]") except OSError: self.logger.warn("Could Not Delete Folder [" + mp3Folder + "]") pass self.session.commit() gc.collect() except: self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions") self.session.rollback() if releaseFolder: scanner.scan(releaseFolder, artist, release) # Sync the counts as some release media and release tracks where added by the processor release.mediaCount = len(release.media) release.trackCount = 0 for media in release.media: media.trackCount = len(media.tracks) release.trackCount += len(media.tracks) releaseFolder = None self.session.commit() if artistsReleasesProcessed and doValidateArtist: self.logger.info("Validating [" + str(len(artistsReleasesProcessed)) + "] Artists") for artistToValidate, artistReleasesToValidate in artistsReleasesProcessed.items(): artistFolder = self.artistFolder(artistToValidate) try: mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(artistFolder)) except: mp3FolderMtime = None pass if mp3FolderMtime and not self._doProcessFolder(artistFolder, mp3FolderMtime, forceFolderScan): self.logger.info("== Skipping Artist Folder [" + artistFolder + "] No Changes Detected") continue for artistReleaseToValidate in artistReleasesToValidate: validator.validate(artistToValidate, artistReleaseToValidate) elapsedTime = arrow.utcnow().datetime - startTime self.logger.info("Processing Complete. Elapsed Time [" + str(elapsedTime) + "]") except: self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions") self.session.rollback()