def processArtists(self, dontValidate):
"""
:param dontValidate: bool
"""
validator = Validator(self.config, self.conn, self.session, False)
for artist in self.session.query(Artist).filter(Artist.isLocked == 0).order_by(Artist.name):
self.logger.debug("=: Processing Artist [" + str(artist.name) + "] RoadieId [" + artist.roadieId + "]")
self.process(folder=self.artistFolder(artist), forceFolderScan=True)
if not dontValidate:
validator.validate(artist)
def process(self, **kwargs):
"""
Process folder using the passed folder
"""
try:
inboundFolder = kwargs.pop('folder', self.InboundFolder)
forceFolderScan = kwargs.pop('forceFolderScan', False)
isReleaseFolder = kwargs.pop('isReleaseFolder', False)
doValidateArtist = kwargs.pop('doValidateArtist', True)
self.logger.info("Processing Folder [" + inboundFolder + "] Flush [" + str(self.flushBefore) + "]")
scanner = Scanner(self.config, self.conn, self.session, self.artistFactory, self.readOnly)
startTime = arrow.utcnow().datetime
newMp3Folder = None
lastID3Artist = None
lastID3Album = None
artist = None
release = None
mp3FoldersProcessed = []
artistsReleasesProcessed = defaultdict(list)
validator = Validator(self.config, self.conn, self.session, self.readOnly)
releaseFolder = None
# Get all the folder in the InboundFolder
for mp3Folder in ProcessorBase.allDirectoriesInDirectory(inboundFolder, isReleaseFolder):
try:
try:
mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(mp3Folder))
except:
mp3FolderMtime = None
pass
if mp3FolderMtime and not self._doProcessFolder(mp3Folder, mp3FolderMtime, forceFolderScan):
self.logger.info("Skipping Folder [" + mp3Folder + "] No Changes Detected")
continue
foundMp3Files = 0
# Do any conversions
if not self.readOnly:
Convertor(mp3Folder)
# Delete any empty folder if enabled
try:
if not os.listdir(mp3Folder) and not self.dontDeleteInboundFolders:
try:
self.logger.warn("X Deleted Empty Folder [" + mp3Folder + "]")
if not self.readOnly:
os.rmdir(mp3Folder)
except OSError:
self.logger.error("Error Deleting [" + mp3Folder + "]")
continue
except:
pass
# Get all the MP3 files in the Folder and process
for rootFolder, mp3 in ProcessorBase.folderMp3Files(mp3Folder):
printableMp3 = mp3.encode('ascii', 'ignore').decode('utf-8')
self.logger.debug("Processing MP3 File [" + printableMp3 + "]")
id3StartTime = arrow.utcnow().datetime
id3 = ID3(mp3, self.processingOptions)
id3ElapsedTime = arrow.utcnow().datetime - id3StartTime
if id3 is not None:
if not id3.isValid():
self.logger.warn("! Track Has Invalid or Missing ID3 Tags [" + printableMp3 + "]")
else:
foundMp3Files += 1
# Get Artist
if lastID3Artist != id3.getReleaseArtist():
artist = None
if not artist:
lastID3Artist = id3.getReleaseArtist()
r = release
if not r:
r = Release()
r.title = id3.album
r.artist = Artist()
r.artist.name = id3.getReleaseArtist()
artist = self.artistFactory.get(id3.getReleaseArtist(), not r.isCastRecording())
if artist and artist.isLocked:
self.logger.debug(
"Skipping Processing Track [" + printableMp3 + "], Artist [" + str(
artist) + "] Is Locked")
continue
if self.flushBefore:
if artist.isLocked:
self.logger.debug(
"Skipping Flushing Artist [" + printableMp3 + "], Artist [" + str(
artist) + "] Is Locked")
continue
else:
for release in artist.releases:
release.genres = []
self.session.delete(release)
self.session.commit()
if not artist:
self.logger.warn(
"! Unable to Find Artist [" + id3.getReleaseArtist() + "] for Mp3 [" + printableMp3 + "]")
continue
# Get the Release
if lastID3Album != id3.album:
release = None
if not release:
lastID3Album = id3.album
release = self.releaseFactory.get(artist, id3.album)
if release:
# Was found now see if needs update based on id3 tag info
id3ReleaseDate = parseDate(id3.year)
if not release.releaseDate == id3ReleaseDate and id3ReleaseDate:
release.releaseDate = id3ReleaseDate
if id3.imageBytes and not release.thumbnail:
try:
img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB')
img.thumbnail(self.thumbnailSize)
b = io.BytesIO()
img.save(b, "JPEG")
release.thumbnail = b.getvalue()
except:
pass
else:
# Was not found in any Searcher create and add
self.logger.debug("Release [" + id3.album + "] Not Found By Factory")
release = self.releaseFactory.create(artist,
string.capwords(id3.album),
1,
id3.year)
if not release:
self.logger.warn("! Unable to Create Album [" + id3.album +
"] For Track [" + printableMp3 + "]")
continue
if release:
if id3.imageBytes:
try:
img = Image.open(io.BytesIO(id3.imageBytes)).convert('RGB')
img.thumbnail(self.thumbnailSize)
b = io.BytesIO()
img.save(b, "JPEG")
release.thumbnail = b.getvalue()
except:
pass
release.status = 1
self.releaseFactory.add(release)
self.logger.info(
"+ Processor Added Release [" + str(release.info()) + "]")
self.session.commit()
if self.shouldMoveToLibrary(artist, id3, mp3):
newMp3 = self.moveToLibrary(artist, id3, mp3)
head, tail = os.path.split(newMp3)
newMp3Folder = head
if artist and release:
if not release.releaseDate and release.media:
for media in release.media:
if media.tracks:
for track in media.tracks:
if track.filePath:
release.releaseDate = parseDate(track.filePath.split('\\')[1][1:5])
break
else:
continue
break
else:
continue
break
else:
continue
break
if not release.releaseDate:
release.releaseDate = parseDate(id3.year)
releaseFolder = self.albumFolder(artist, release.releaseDate.strftime('%Y'), release.title)
if newMp3Folder and newMp3Folder not in mp3FoldersProcessed:
for coverImage in self.releaseCoverImages(mp3Folder):
try:
im = Image.open(coverImage).convert('RGB')
newPath = os.path.join(newMp3Folder, "cover.jpg")
if (not os.path.isfile(newPath) or not os.path.samefile(coverImage,
newPath)) and not self.readOnly:
im.save(newPath)
self.logger.info(
"+ Copied Cover File [" + coverImage + "] => [" + newPath + "]")
except:
self.logger.exception("Error Copying File [" + coverImage + "]")
pass
mp3FoldersProcessed.append(newMp3Folder)
if release not in artistsReleasesProcessed[artist]:
artistsReleasesProcessed[artist].append(release)
if not self.readOnly and artist and release:
if self.shouldDeleteFolder(mp3Folder):
try:
shutil.rmtree(mp3Folder)
self.logger.debug("x Deleted Processed Folder [" + mp3Folder + "]")
except OSError:
self.logger.warn("Could Not Delete Folder [" + mp3Folder + "]")
pass
self.session.commit()
gc.collect()
except:
self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions")
self.session.rollback()
if releaseFolder:
scanner.scan(releaseFolder, artist, release)
# Sync the counts as some release media and release tracks where added by the processor
release.mediaCount = len(release.media)
release.trackCount = 0
for media in release.media:
media.trackCount = len(media.tracks)
release.trackCount += len(media.tracks)
releaseFolder = None
self.session.commit()
if artistsReleasesProcessed and doValidateArtist:
self.logger.info("Validating [" + str(len(artistsReleasesProcessed)) + "] Artists")
for artistToValidate, artistReleasesToValidate in artistsReleasesProcessed.items():
artistFolder = self.artistFolder(artistToValidate)
try:
mp3FolderMtime = max(os.path.getmtime(root) for root, _, _ in os.walk(artistFolder))
except:
mp3FolderMtime = None
pass
if mp3FolderMtime and not self._doProcessFolder(artistFolder, mp3FolderMtime, forceFolderScan):
self.logger.info("== Skipping Artist Folder [" + artistFolder + "] No Changes Detected")
continue
for artistReleaseToValidate in artistReleasesToValidate:
validator.validate(artistToValidate, artistReleaseToValidate)
elapsedTime = arrow.utcnow().datetime - startTime
self.logger.info("Processing Complete. Elapsed Time [" + str(elapsedTime) + "]")
except:
self.logger.exception("Processing Exception Occurred, Rolling Back Session Transactions")
self.session.rollback()
help='Process Artist release folders in Library')
p.add_argument('--flush', '-fl', action='store_true', help='Flush All Releases For Artist Before Processing')
p.add_argument('--readOnly', '-st', action='store_true', help='Read Only Mode; Dont modify Anything')
args = p.parse_args()
d = os.path.dirname(os.path.realpath(__file__)).split(os.sep)
path = os.path.join(os.sep.join(d), "settings.json")
with open(path, "r") as rf:
config = json.load(rf)
forceFolderScan = False
if args.folder:
config['ROADIE_INBOUND_FOLDER'] = args.folder
forceFolderScan = True
engine = create_engine(config['ROADIE_DATABASE_URL'])
conn = engine.connect()
Base = declarative_base()
Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
session = DBSession()
pp = Processor(config, conn, session, args.readOnly, args.dontDeleteInboundFolders, args.flush)
if args.processArtists:
pp.processArtists(args.dontValidate)
else:
pp.process(forceFolderScan=forceFolderScan)
if args.validate:
validator = Validator(config, conn, session, False)
validator.validateArtists()