def __init__(self, dbConn, dbSession, readOnly):
     self.logger = Logger()
     self.dbConn = dbConn
     self.dbSession = dbSession
     self.artistFactory = ArtistFactory(dbConn, dbSession)
     self.releaseFactory = ReleaseFactory(dbConn, dbSession)
     self.notFoundEntryInfo = []
     self.readOnly = readOnly
Exemple #2
0
 def __init__(self, config, dbConn, dbSession, readOnly, dontDeleteInboundFolders, flushBefore=False, logger=None):
     self.config = config
     self.InboundFolder = self.config['ROADIE_INBOUND_FOLDER']
     # TODO if set then process music files; like clear comments
     self.processingOptions = self.config['ROADIE_PROCESSING']
     self.conn = dbConn
     self.session = dbSession
     self.thumbnailSize = self.config['ROADIE_THUMBNAILS']['Height'], self.config['ROADIE_THUMBNAILS']['Width']
     self.readOnly = readOnly or False
     self.logger = logger or Logger()
     self.dontDeleteInboundFolders = dontDeleteInboundFolders or False
     self.flushBefore = flushBefore
     self.artistFactory = ArtistFactory(dbConn, dbSession)
     self.releaseFactory = ReleaseFactory(dbConn, dbSession)
     self.folderDB = sqlite3.connect("processorFolder.db")
     self.folderDB.execute("CREATE TABLE IF NOT EXISTS `folder` (namehash TEXT, mtime REAL);")
     self.folderDB.execute("CREATE INDEX IF NOT EXISTS `idx_folder_namehash` ON `folder`(namehash);")
     self.folderDB.commit()
     super().__init__(config, self.logger)
Exemple #3
0
class Processor(ProcessorBase):
    def __init__(self, config, dbConn, dbSession, readOnly, dontDeleteInboundFolders, flushBefore=False, logger=None):
        self.config = config
        self.InboundFolder = self.config['ROADIE_INBOUND_FOLDER']
        # TODO if set then process music files; like clear comments
        self.processingOptions = self.config['ROADIE_PROCESSING']
        self.conn = dbConn
        self.session = dbSession
        self.thumbnailSize = self.config['ROADIE_THUMBNAILS']['Height'], self.config['ROADIE_THUMBNAILS']['Width']
        self.readOnly = readOnly or False
        self.logger = logger or Logger()
        self.dontDeleteInboundFolders = dontDeleteInboundFolders or False
        self.flushBefore = flushBefore
        self.artistFactory = ArtistFactory(dbConn, dbSession)
        self.releaseFactory = ReleaseFactory(dbConn, dbSession)
        self.folderDB = sqlite3.connect("processorFolder.db")
        self.folderDB.execute("CREATE TABLE IF NOT EXISTS `folder` (namehash TEXT, mtime REAL);")
        self.folderDB.execute("CREATE INDEX IF NOT EXISTS `idx_folder_namehash` ON `folder`(namehash);")
        self.folderDB.commit()
        super().__init__(config, self.logger)

    def _doProcessFolder(self, name, mtime, forceFolderScan):
        try:
            nameHash = hashlib.md5(name.encode('utf-8')).hexdigest()
            folderDbRecord = self.folderDB.execute(
                "SELECT * FROM `folder` WHERE namehash='" + nameHash + "';").fetchone()
            if not folderDbRecord:
                self.folderDB.execute("INSERT INTO `folder` VALUES ('" + nameHash + "', " + str(mtime) + ");")
                self.folderDB.commit()
                return True
            if forceFolderScan:
                if folderDbRecord:
                    self.folderDB.execute(
                        "UPDATE `folder` SET mtime = " + str(mtime) + " WHERE namehash = '" + nameHash + "';")
                else:
                    self.folderDB.execute("INSERT INTO `folder` VALUES ('" + nameHash + "', " + str(mtime) + ");")
                self.folderDB.commit()
                return True
            return folderDbRecord[1] != mtime
        except:
            return True

    def releaseCoverImages(self, folder):
        try:
            image_filter = ['.jpg', '.jpeg', '.bmp', '.png', '.gif']
            cover_filter = ['cover', 'front']
            for r, d, f in os.walk(folder):
                for file in f:
                    root, file = os.path.split(file)
                    root, ext = os.path.splitext(file)
                    if ext.lower() in image_filter and root.lower() in cover_filter:
                        yield os.path.join(r, file)
        except:
            self.logger.exception()

    def shouldDeleteFolder(self, mp3Folder):
        if self.dontDeleteInboundFolders:
            return False

        if self.readOnly:
            return False

        # Is folder to delete empty?
        if not os.listdir(mp3Folder):
            return True

        return False

    # Determine if the found file should be moved into the library; check for existing and see if better
    def shouldMoveToLibrary(self, artist, id3, mp3):
        try:
            fileFolderLibPath = self.albumFolder(artist, id3.year, id3.album)
            os.makedirs(fileFolderLibPath, exist_ok=True)
            fullFileLibPath = os.path.join(fileFolderLibPath,
                                           ProcessorBase.makeFileFriendly(
                                               ProcessorBase.trackName(id3.track, id3.title)))
            if not os.path.isfile(fullFileLibPath):
                # Does not exist copy it over
                return True
            else:
                # Does exist see if the one being copied is 'better' then the existing
                existingId3 = ID3(fullFileLibPath, self.processingOptions)
                if not existingId3.isValid():
                    return True
                existingId3Hash = hashlib.md5((str(artist.roadieId) + str(existingId3)).encode('utf-8')).hexdigest()
                id3Hash = hashlib.md5((str(artist.roadieId) + str(id3)).encode('utf-8')).hexdigest()
                if existingId3Hash == id3Hash:
                    # If the hashes are equal its Likely the same file
                    return False
                # If The existing is longer or has a higher bitrate then use existing
                if existingId3.length > id3.length and existingId3.bitrate > id3.bitrate:
                    return False
            return True
        except:
            self.logger.exception("shouldMoveToLibrary: Id3 [" + str(id3) + "]")
            return False

    def processArtists(self, dontValidate):
        """

        :param dontValidate: bool
        """
        validator = Validator(self.config, self.conn, self.session, False)
        for artist in self.session.query(Artist).filter(Artist.isLocked == 0).order_by(Artist.name):
            self.logger.debug("=: Processing Artist [" + str(artist.name) + "] RoadieId [" + artist.roadieId + "]")
            self.process(folder=self.artistFolder(artist), forceFolderScan=True)
            if not dontValidate:
                validator.validate(artist)

    def process(self, **kwargs):
        """
        Process folder using the passed folder

        """
        try:
            inboundFolder = kwargs.pop('folder', self.InboundFolder)
            forceFolderScan = kwargs.pop('forceFolderScan', False)
            isReleaseFolder = kwargs.pop('isReleaseFolder', False)
            doValidateArtist = kwargs.pop('doValidateArtist', True)
            self.logger.info("Processing Folder [" + inboundFolder + "] Flush [" + str(self.flushBefore) + "]")
            scanner = Scanner(self.config, self.conn, self.session, self.artistFactory, self.readOnly)
            startTime = arrow.utcnow().datetime
            newMp3Folder = None
            lastID3Artist = None
            lastID3Album = None
            artist = None
            release = None
            mp3FoldersProcessed = []
            artistsReleasesProcessed = defaultdict(list)
            validator = Validator(self.config, self.conn, self.session, self.readOnly)
            releaseFolder = None
            # Get all the folder in the InboundFolder
            for mp3Folder in ProcessorBase.allDirectoriesInDirectory(inboundFolder, isReleaseFolder):
                try:
                    try:
                        mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(mp3Folder))
                    except:
                        mp3FolderMtime = None
                        pass
                    if mp3FolderMtime and not self._doProcessFolder(mp3Folder, mp3FolderMtime, forceFolderScan):
                        self.logger.info("Skipping Folder [" + mp3Folder + "] No Changes Detected")
                        continue

                    foundMp3Files = 0

                    # Do any conversions
                    if not self.readOnly:
                        Convertor(mp3Folder)

                    # Delete any empty folder if enabled
                    try:
                        if not os.listdir(mp3Folder) and not self.dontDeleteInboundFolders:
                            try:
                                self.logger.warn("X Deleted Empty Folder [" + mp3Folder + "]")
                                if not self.readOnly:
                                    os.rmdir(mp3Folder)
                            except OSError:
                                self.logger.error("Error Deleting [" + mp3Folder + "]")
                            continue
                    except:
                        pass

                    # Get all the MP3 files in the Folder and process
                    for rootFolder, mp3 in ProcessorBase.folderMp3Files(mp3Folder):
                        printableMp3 = mp3.encode('ascii', 'ignore').decode('utf-8')
                        self.logger.debug("Processing MP3 File [" + printableMp3 + "]")
                        id3StartTime = arrow.utcnow().datetime
                        id3 = ID3(mp3, self.processingOptions)
                        id3ElapsedTime = arrow.utcnow().datetime - id3StartTime
                        if id3 is not None:
                            if not id3.isValid():
                                self.logger.warn("! Track Has Invalid or Missing ID3 Tags [" + printableMp3 + "]")
                            else:
                                foundMp3Files += 1
                                # Get Artist
                                if lastID3Artist != id3.getReleaseArtist():
                                    artist = None
                                if not artist:
                                    lastID3Artist = id3.getReleaseArtist()
                                    r = release
                                    if not r:
                                        r = Release()
                                        r.title = id3.album
                                        r.artist = Artist()
                                        r.artist.name = id3.getReleaseArtist()
                                    artist = self.artistFactory.get(id3.getReleaseArtist(), not r.isCastRecording())
                                if artist and artist.isLocked:
                                    self.logger.debug(
                                        "Skipping Processing Track [" + printableMp3 + "], Artist [" + str(
                                            artist) + "] Is Locked")
                                    continue
                                if self.flushBefore:
                                    if artist.isLocked:
                                        self.logger.debug(
                                            "Skipping Flushing Artist [" + printableMp3 + "], Artist [" + str(
                                                artist) + "] Is Locked")
                                        continue
                                    else:
                                        for release in artist.releases:
                                            release.genres = []
                                            self.session.delete(release)
                                        self.session.commit()
                                if not artist:
                                    self.logger.warn(
                                        "! Unable to Find Artist [" + id3.getReleaseArtist() + "] for Mp3 [" + printableMp3 + "]")
                                    continue
                                # Get the Release
                                if lastID3Album != id3.album:
                                    release = None
                                if not release:
                                    lastID3Album = id3.album
                                    release = self.releaseFactory.get(artist, id3.album)
                                    if release:
                                        # Was found now see if needs update based on id3 tag info
                                        id3ReleaseDate = parseDate(id3.year)
                                        if not release.releaseDate == id3ReleaseDate and id3ReleaseDate:
                                            release.releaseDate = id3ReleaseDate
                                        if id3.imageBytes and not release.thumbnail:
                                            try:
                                                img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB')
                                                img.thumbnail(self.thumbnailSize)
                                                b = io.BytesIO()
                                                img.save(b, "JPEG")
                                                release.thumbnail = b.getvalue()
                                            except:
                                                pass

                                    else:
                                        # Was not found in any Searcher create and add
                                        self.logger.debug("Release [" + id3.album + "] Not Found By Factory")
                                        release = self.releaseFactory.create(artist,
                                                                             string.capwords(id3.album),
                                                                             1,
                                                                             id3.year)
                                        if not release:
                                            self.logger.warn("! Unable to Create Album [" + id3.album +
                                                             "] For Track [" + printableMp3 + "]")
                                            continue
                                        if release:
                                            if id3.imageBytes:
                                                try:
                                                    img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB')
                                                    img.thumbnail(self.thumbnailSize)
                                                    b = io.BytesIO()
                                                    img.save(b, "JPEG")
                                                    release.thumbnail = b.getvalue()
                                                except:
                                                    pass
                                            release.status = 1
                                            self.releaseFactory.add(release)
                                            self.logger.info(
                                                "+ Processor Added Release [" + str(release.info()) + "]")
                                            self.session.commit()
                                if self.shouldMoveToLibrary(artist, id3, mp3):
                                    newMp3 = self.moveToLibrary(artist, id3, mp3)
                                    head, tail = os.path.split(newMp3)
                                    newMp3Folder = head
                    if artist and release:
                        if not release.releaseDate and release.media:
                            for media in release.media:
                                if media.tracks:
                                    for track in media.tracks:
                                        if track.filePath:
                                            release.releaseDate = parseDate(track.filePath.split('\\')[1][1:5])
                                            break
                                        else:
                                            continue
                                        break
                                    else:
                                        continue
                                    break
                            else:
                                continue
                            break
                        if not release.releaseDate:
                            release.releaseDate = parseDate(id3.year)
                        releaseFolder = self.albumFolder(artist, release.releaseDate.strftime('%Y'), release.title)
                        if newMp3Folder and newMp3Folder not in mp3FoldersProcessed:
                            for coverImage in self.releaseCoverImages(mp3Folder):
                                try:
                                    im = Image.open(coverImage).convert('RGB')
                                    newPath = os.path.join(newMp3Folder, "cover.jpg")
                                    if (not os.path.isfile(newPath) or not os.path.samefile(coverImage,
                                                                                            newPath)) and not self.readOnly:
                                        im.save(newPath)
                                        self.logger.info(
                                            "+ Copied Cover File [" + coverImage + "] => [" + newPath + "]")
                                except:
                                    self.logger.exception("Error Copying File [" + coverImage + "]")
                                    pass
                            mp3FoldersProcessed.append(newMp3Folder)
                        if release not in artistsReleasesProcessed[artist]:
                            artistsReleasesProcessed[artist].append(release)
                    if not self.readOnly and artist and release:
                        if self.shouldDeleteFolder(mp3Folder):
                            try:
                                shutil.rmtree(mp3Folder)
                                self.logger.debug("x Deleted Processed Folder [" + mp3Folder + "]")
                            except OSError:
                                self.logger.warn("Could Not Delete Folder [" + mp3Folder + "]")
                                pass

                    self.session.commit()
                    gc.collect()

                except:
                    self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions")
                    self.session.rollback()
                if releaseFolder:
                    scanner.scan(releaseFolder, artist, release)
                    # Sync the counts as some release media and release tracks where added by the processor
                    release.mediaCount = len(release.media)
                    release.trackCount = 0
                    for media in release.media:
                        media.trackCount = len(media.tracks)
                        release.trackCount += len(media.tracks)
                    releaseFolder = None
            self.session.commit()
            if artistsReleasesProcessed and doValidateArtist:
                self.logger.info("Validating [" + str(len(artistsReleasesProcessed)) + "] Artists")
                for artistToValidate, artistReleasesToValidate in artistsReleasesProcessed.items():
                    artistFolder = self.artistFolder(artistToValidate)
                    try:
                        mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(artistFolder))
                    except:
                        mp3FolderMtime = None
                        pass
                    if mp3FolderMtime and not self._doProcessFolder(artistFolder, mp3FolderMtime, forceFolderScan):
                        self.logger.info("== Skipping Artist Folder [" + artistFolder + "] No Changes Detected")
                        continue
                    for artistReleaseToValidate in artistReleasesToValidate:
                        validator.validate(artistToValidate, artistReleaseToValidate)
            elapsedTime = arrow.utcnow().datetime - startTime
            self.logger.info("Processing Complete. Elapsed Time [" + str(elapsedTime) + "]")
        except:
            self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions")
            self.session.rollback()
class CollectionImporter(ProcessorBase):
    format = None
    positions = None
    filename = None
    collectionId = None
    collection = None

    def __init__(self, dbConn, dbSession, readOnly):
        self.logger = Logger()
        self.dbConn = dbConn
        self.dbSession = dbSession
        self.artistFactory = ArtistFactory(dbConn, dbSession)
        self.releaseFactory = ReleaseFactory(dbConn, dbSession)
        self.notFoundEntryInfo = []
        self.readOnly = readOnly

    def _findColumns(self):
        self.position = -1
        self.release = -1
        self.artist = -1
        for i, position in enumerate(self.positions):
            if position.lower() == "position":
                self.position = i
            elif position.lower() == "release" or position.lower() == "album":
                self.release = i
            elif position.lower() == "artist":
                self.artist = i
        if self.position < 0 or self.release < 0 or self.artist < 0:
            self.logger.critical("Unable To Find Required Positions")
            return False
        return True

    def importFile(self, collectionId, fileFormat, filename):
        self.collectionId = collectionId
        self.collection = self.dbSession.query(Collection).filter(Collection.id == collectionId).first()
        self.format = fileFormat
        self.positions = self.format.split(',')
        self.filename = filename
        if not os.path.exists(self.filename):
            self.logger.critical("Unable to Find CSV File [" + self.filename + "]")
        else:
            self.logger.debug("Importing [" + self.filename + "]")
            return self.importCsvData(open(self.filename))

    def importCollection(self, collection):
        self.collectionId = collection.id
        self.collection = collection
        self.positions = collection.listInCSVFormat.split(',')
        self.importCsvData(io.StringIO(collection.listInCSV))

    def importCsvData(self, csvData):
        try:
            if not self.collection:
                self.logger.critical("Unable to Find Collection Id [" + self.collectionId + "]")
                return False
            self._findColumns()
            reader = csv.reader(csvData)
            self.collection.collectionReleases = []
            for row in reader:
                csvPosition = int(row[self.position].strip())
                csvArtist = row[self.artist].strip()
                csvRelease = row[self.release].strip()
                artist = self.artistFactory.get(csvArtist, False)
                if not artist:
                    self.logger.warn(("Artist [" + csvArtist + "] Not Found In Database").encode('utf-8'))
                    self.notFoundEntryInfo.append(
                        {'col': self.collection.name, 'position': csvPosition, 'artist': csvArtist,
                         'release': csvRelease});
                    continue
                release = self.releaseFactory.get(artist, csvRelease, False)
                if not release:
                    self.logger.warn(
                        ("Not able to find Release [" + csvRelease + "], Artist [" + csvArtist + "]").encode(
                            'utf-8'))
                    self.notFoundEntryInfo.append(
                        {'col': self.collection.name, 'position': csvPosition, 'artist': csvArtist,
                         'release': csvRelease})
                    continue
                colRelease = CollectionRelease()
                colRelease.releaseId = release.id
                colRelease.listNumber = csvPosition
                colRelease.createdDate = arrow.utcnow().datetime
                colRelease.roadieId = str(uuid.uuid4())
                self.collection.collectionReleases.append(colRelease)
                self.logger.info(
                    "Added Position [" + str(csvPosition) + "] Release [" + str(release) + "] To Collection")
            self.collection.lastUpdated = arrow.utcnow().datetime
            self.dbSession.commit()
            return True
        except:
            self.logger.exception("Error Importing Collection [" + self.collection.name + "]")
            self.dbSession.rollback()
            return False