class Validator(ProcessorBase):
def __init__(self, config, dbConn, dbSession, readOnly):
self.readOnly = readOnly or False
self.logger = Logger()
self.conn = dbConn
self.session = dbSession
super().__init__(config, self.logger)
def validateArtists(self):
for artist in self.session.query(Artist).all():
self.validate(artist)
def validate(self, artist, onlyValidateRelease=None):
"""
Do sanity checks on given Artist
:param onlyValidateRelease: Release
:param artist: Artist
:return:
"""
if not artist:
raise RuntimeError("Invalid Artist")
if not self.config:
raise RuntimeError("Invalid Configuration")
if not self.libraryFolder:
raise RuntimeError("Invalid Configuration: Library Folder Not Set")
now = arrow.utcnow().datetime
self.logger.info("Validating Artist [" + artist.name + "] Release Set [" + str(
onlyValidateRelease is None) + "]")
try:
for release in artist.releases:
issuesFound = False
if onlyValidateRelease and release.roadieId != onlyValidateRelease.roadieId:
continue
releaseFolder = self.albumFolder(artist, release.releaseDate.strftime('%Y'), release.title)
try:
folderExists = os.path.exists(releaseFolder)
except:
folderExists = False
if not folderExists:
if not self.readOnly:
for media in release.media:
for track in media.tracks:
locatedTrackInfo = self.tryToFindFileForTrack(artist, track)
trackFullPath = track.fullPath()
if locatedTrackInfo and trackFullPath and not os.path.exists(trackFullPath):
movedFile = self.moveToLibrary(artist, locatedTrackInfo['id3'],
locatedTrackInfo['fileName'])
if movedFile:
self.logger.warn(
"! Moved File From [" + locatedTrackInfo[
'fileName'] + "] To [" + movedFile + "]")
folderExists = True
else:
track.filePath = None
track.fileName = None
track.fileSize = 0
track.hash = None
track.lastUpdated = now
if not folderExists:
release.libraryStatus = 'Incomplete'
self.logger.warn(
"X Marking Release Missing [" + str(
release) + "] Missing Folder [" + releaseFolder + "] Not Found")
continue
releaseTrackCount = 0
# If release is not already complete, set to complete unless its found otherwise
release.libraryStatus = 'Complete'
releaseMediaWithTracks = []
for releaseMedia in release.media:
releaseMediaTrackCount = 0
for track in sorted(releaseMedia.tracks, key=lambda tt: tt.trackNumber):
try:
trackFilename = self.pathToTrack(track)
isTrackFilePresent = False
if trackFilename:
try:
isTrackFilePresent = os.path.isfile(trackFilename)
except:
self.logger.exception()
pass
if not isTrackFilePresent:
# See if track exists in another folder and title was renamed so folder no longer
# matches what it is expected to be
if not self.readOnly:
locatedTrackInfo = self.tryToFindFileForTrack(artist, track)
if locatedTrackInfo and not isEqual(trackFilename, locatedTrackInfo['fileName']):
movedFile = self.moveToLibrary(artist, locatedTrackInfo['id3'],
locatedTrackInfo['fileName'])
if movedFile:
head, tail = os.path.split(movedFile)
headNoLibrary = head.replace(self.config['ROADIE_LIBRARY_FOLDER'], "")
trackHash = self.makeTrackHash(artist.roadieId, movedFile)
track.fileName = tail
track.filePath = headNoLibrary
track.hash = trackHash
track.fileSize = os.path.getsize(movedFile)
track.lastUpdated = now
self.logger.warn(
"! Located Track [" + str(track.info(includePathInfo=True)) + "]")
isTrackFilePresent = True
else:
track.filePath = None
track.fileName = None
track.fileSize = 0
track.hash = None
track.lastUpdated = now
self.logger.warn(
"X Missing Track [" + str(
track.info(includePathInfo=True)) + "] File [" + str(
trackFilename) + "]")
issuesFound = True
release.libraryStatus = 'Incomplete'
if isTrackFilePresent:
releaseMediaTrackCount += 1
releaseTrackCount += 1
if not isEqual(track.trackNumber, releaseMediaTrackCount):
self.logger.warn("! Track Number Sequence Incorrect Is [" +
str(track.trackNumber) + "] Expected [" +
str(releaseMediaTrackCount) + "]")
release.libraryStatus = 'Incomplete'
issuesFound = True
except:
self.logger.exception()
issuesFound = True
pass
releaseMedia.trackCount = releaseMediaTrackCount
if releaseMedia.trackCount > 0:
releaseMediaWithTracks.append(releaseMedia)
if not self.readOnly:
release.media = releaseMediaWithTracks
release.mediaCount = len(releaseMediaWithTracks)
# Seems not likely that a release only has a single track; more likely missing unknown tracks
if releaseTrackCount > 1 and release.trackCount < 2:
release.trackCount = releaseTrackCount
release.lastUpdated = now
self.logger.info("Validated Artist [" + str(artist) + "], " +
"Release [" + str(release) + "], " +
"IssuesFound [" + str(issuesFound) + "]")
if not self.readOnly:
self.session.commit()
else:
self.session.rollback()
except:
self.logger.exception("Validating Artist, Rolling Back Session Transactions")
try:
self.session.rollback()
except:
pass
class CollectionImporter(ProcessorBase):
format = None
positions = None
filename = None
collectionId = None
collection = None
def __init__(self, dbConn, dbSession, readOnly):
self.logger = Logger()
self.dbConn = dbConn
self.dbSession = dbSession
self.artistFactory = ArtistFactory(dbConn, dbSession)
self.releaseFactory = ReleaseFactory(dbConn, dbSession)
self.notFoundEntryInfo = []
self.readOnly = readOnly
def _findColumns(self):
self.position = -1
self.release = -1
self.artist = -1
for i, position in enumerate(self.positions):
if position.lower() == "position":
self.position = i
elif position.lower() == "release" or position.lower() == "album":
self.release = i
elif position.lower() == "artist":
self.artist = i
if self.position < 0 or self.release < 0 or self.artist < 0:
self.logger.critical("Unable To Find Required Positions")
return False
return True
def importFile(self, collectionId, fileFormat, filename):
self.collectionId = collectionId
self.collection = self.dbSession.query(Collection).filter(Collection.id == collectionId).first()
self.format = fileFormat
self.positions = self.format.split(',')
self.filename = filename
if not os.path.exists(self.filename):
self.logger.critical("Unable to Find CSV File [" + self.filename + "]")
else:
self.logger.debug("Importing [" + self.filename + "]")
return self.importCsvData(open(self.filename))
def importCollection(self, collection):
self.collectionId = collection.id
self.collection = collection
self.positions = collection.listInCSVFormat.split(',')
self.importCsvData(io.StringIO(collection.listInCSV))
def importCsvData(self, csvData):
try:
if not self.collection:
self.logger.critical("Unable to Find Collection Id [" + self.collectionId + "]")
return False
self._findColumns()
reader = csv.reader(csvData)
self.collection.collectionReleases = []
for row in reader:
csvPosition = int(row[self.position].strip())
csvArtist = row[self.artist].strip()
csvRelease = row[self.release].strip()
artist = self.artistFactory.get(csvArtist, False)
if not artist:
self.logger.warn(("Artist [" + csvArtist + "] Not Found In Database").encode('utf-8'))
self.notFoundEntryInfo.append(
{'col': self.collection.name, 'position': csvPosition, 'artist': csvArtist,
'release': csvRelease});
continue
release = self.releaseFactory.get(artist, csvRelease, False)
if not release:
self.logger.warn(
("Not able to find Release [" + csvRelease + "], Artist [" + csvArtist + "]").encode(
'utf-8'))
self.notFoundEntryInfo.append(
{'col': self.collection.name, 'position': csvPosition, 'artist': csvArtist,
'release': csvRelease})
continue
colRelease = CollectionRelease()
colRelease.releaseId = release.id
colRelease.listNumber = csvPosition
colRelease.createdDate = arrow.utcnow().datetime
colRelease.roadieId = str(uuid.uuid4())
self.collection.collectionReleases.append(colRelease)
self.logger.info(
"Added Position [" + str(csvPosition) + "] Release [" + str(release) + "] To Collection")
self.collection.lastUpdated = arrow.utcnow().datetime
self.dbSession.commit()
return True
except:
self.logger.exception("Error Importing Collection [" + self.collection.name + "]")
self.dbSession.rollback()
return False
class ID3(object):
filename = None
config = None
def __init__(self, path, config=None):
self.logger = Logger()
self.filename = path
self.config = config
self._load(path, config)
def isValid(self):
try:
if self.artist and \
self.year and \
self.album and \
self.track and \
self.title and \
self.bitrate and \
self.length > 0:
return True
else:
return False
except:
return False
def info(self):
return "--- IsValid: [" + str(self.isValid()) + "] " + \
"Artist [" + str(self.getArtist()) + "] " + \
"HasTrackArtist [" + str(self.hasTrackArtist()) + "] " + \
"Artist (TPE1) [" + str(self.artist) + "], " + \
"Album Artist (TPE2) [" + str(self.albumArtist) + "], " + \
"Year [" + str(self.year) + "], " + \
"Album: [" + str(self.album) + "], " + \
"Disc: [" + str(self.disc) + "], " + \
"Track [" + str(self.track).zfill(2) + "], " + \
"Title [" + str(self.title) + "], " + \
"(" + str(self.bitrate) + "bps::" + str(self.length) + ")"
def __str__(self):
return str(self.artist) + "." + \
str(self.year) + "." + \
str(self.album) + "." + \
str(self.track) + "." + \
str(self.title) + "." + \
str(self.bitrate) + "." + \
str(self.length)
def setCoverImage(self, image):
try:
tags = mutagenID3(self.filename)
except ID3NoHeaderError:
tags = mutagenID3()
if self.config:
if 'DoClearComments' in self.config:
if self.config['DoClearComments'].lower() == "true":
tags.delall(u"COMM::'en'")
tags.delall(u"APIC::'en'")
tags.add(APIC(
encoding=3,
mime='image/jpeg',
type=3,
desc=u'Cover',
data=image
))
tags.save(self.filename, v2_version=3) # this is for Windows Media Player compatibility
def updateFromTrack(self, track):
"""
Update the ID3 Track with the given Track values
:param track: Track
:return:
"""
try:
tags = mutagenID3(self.filename)
except ID3NoHeaderError:
tags = mutagenID3()
tags["TIT2"] = TIT2(encoding=3, text=track.title)
if track.artist:
tags["TPE1"] = TPE1(encoding=3, text=track.artist.name)
tags["TRCK"] = TRCK(encoding=3, text=str(track.trackNumber))
if self.config:
if 'DoClearComments' in self.config:
if self.config['DoClearComments'].lower() == "true":
tags.delall(u"COMM::'en'")
tags.save(self.filename)
def updateFromRelease(self, release, track):
"""
Update the given Track with loaded values
:param release: Release
:param track: Track
:return:
"""
try:
tags = mutagenID3(self.filename)
except ID3NoHeaderError:
tags = mutagenID3()
tags["TIT2"] = TIT2(encoding=3, text=track.title)
tags["TALB"] = TALB(encoding=3, text=release.title)
if track.artist:
tags["TPE2"] = TPE2(encoding=3, text=release.artist.name)
tags["TPE1"] = TPE1(encoding=3, text=track.artist.name)
else:
tags["TPE1"] = TPE1(encoding=3, text=release.artist.name)
tags["TRCK"] = TRCK(encoding=3, text=str(track.trackNumber))
if release.releaseDate:
year = release.releaseDate.strftime('%Y')
if year:
tags["TDRC"] = TDRC(encoding=3, text=year)
if self.config:
if 'DoClearComments' in self.config:
if self.config['DoClearComments'].lower() == "true":
tags.delall(u"COMM::'en'")
tags.save(self.filename)
def getTrackArtist(self):
"""
Return the artist to use for this track be it Artist ("TPE1") or Album Artist ("TPE2")
:param self:
:return: str
"""
return (self.artist or '').strip()
def getReleaseArtist(self):
"""
Return the artist to use for this Release be it Artist ("TPE1") or Album Artist ("TPE2")
:param self:
:return: str
"""
if self.hasTrackArtist():
return (self.albumArtist or '').strip()
return (self.artist or '').strip()
def hasTrackArtist(self):
# Artist is always set
artist = (self.artist or '').strip()
# Album Artist is sometimes set and most of the times when set its the same as the Artist
albumArtist = (self.albumArtist or '').strip()
if albumArtist and not isEqual(artist, albumArtist):
return True
return False
def _load(self, filename, config):
self.dirty = False
self.artist = ''
self.artists = []
self.albumArtist = ''
self.album = ''
self.track = ''
self.title = ''
self.year = ''
self.disc = -1
self.bitrate = ''
self.length = -1
try:
short_tags = full_tags = mutagen.File(filename)
comments = []
if isinstance(full_tags, mutagen.mp3.MP3):
for key in short_tags:
if key[0:4] == 'COMM':
if short_tags[key].desc == '':
comments.append(short_tags[key].text[0])
short_tags = mutagen.mp3.MP3(filename, ID3=mutagen.easyid3.EasyID3)
comments.append('')
self.album = string.capwords(short_tags.get('album', [''])[0])
self.artist = string.capwords(short_tags.get('artist', [''])[0])
try:
# id3v2.3.0, 4.2.1 TPE1 [#TPE1 Lead performer(s)/Soloist(s)]
if self.artist and "/" in self.artist:
self.artists = []
for aa in self.artist.split("/"):
if aa:
self.artists.append(string.capwords(aa.strip()))
if len(self.artists) > 0:
self.artist = self.artists[0]
except:
pass
try:
self.albumArtist = full_tags['TPE2'].text[0]
except:
pass
self.duration = "%u:%.2d" % (full_tags.info.length / 60, full_tags.info.length % 60)
trackNumber = short_tags.get('tracknumber', [''])[0]
self.track = 0
try:
if trackNumber and "/" in trackNumber:
self.track = int(trackNumber.split("/")[0])
if trackNumber:
self.track = int(trackNumber)
except:
pass
try:
self.length = full_tags.info.length
except:
pass
try:
self.bitrate = full_tags.info.bitrate
except:
pass
discNumber = short_tags.get('discnumber', [''])[0]
self.disc = 0
try:
if discNumber and "/" in discNumber:
self.disc = int(discNumber.split("/")[0])
elif discNumber:
self.disc = int(discNumber)
except:
pass
self.year = short_tags.get('date', [''])[0]
if not self.year:
myfile = mpeg.Mpeg(filename)
if myfile:
self.year = myfile.tag.year[:4]
try:
if not self.year:
self.year = full_tags.tags._DictProxy__dict['TDRL'].text[0].text
except:
pass
self.title = string.capwords(short_tags.get('title', [''])[0])
if self.title and config:
if 'TitleReplacements' in config:
for rpl in config['TitleReplacements']:
for key, val in rpl.items():
self.title = self.title.replace(key, val)
self.dirty = True
self.title = string.capwords(self.title)
if self.title and self.track:
if self.title.startswith('%02d - ' % self.track):
self.title = self.title[5:]
elif self.title.startswith('%02d ' % self.track):
self.title = self.title[3:]
elif self.title.startswith('- '):
self.title = self.title[2:]
self.title = string.capwords(self.title)
self.dirty = True
self.comment = string.capwords(comments[0])
if self.comment and config:
if 'DoClearComments' in config:
if config['DoClearComments'].lower() == "true":
self.comment = None
self.dirty = True
self.genre = ''
genres = short_tags.get('genre', [''])
if len(genres) > 0:
self.genre = genres[0]
self.imageBytes = None
try:
if full_tags.tags and 'APIC:' in full_tags.tags:
self.imageBytes = full_tags.tags._DictProxy__dict['APIC:'].data
except:
pass
except:
self.logger.exception()
class ReleaseFactory(object):
def __init__(self, dbConn, dbSession):
self.conn = dbConn
self.session = dbSession
self.logger = Logger()
self.searcher = ArtistSearcher()
def getAllForArtist(self, artist, forceRefresh=False):
"""
Query Database for a release with the given title, if not found search and if found save and return results
:param artist: Artist
:param forceRefresh: bool
"""
if not artist:
return None
printableArtistName = artist.name.encode('ascii', 'ignore').decode('utf-8')
releases = self._getAllFromDatabaseForArtist(artist)
if not releases or forceRefresh:
if not releases:
self.logger.info("Releases For Artist [" + printableArtistName + "] Not Found")
else:
self.logger.info("Refreshing Releases For Artist [" + printableArtistName + "]")
releases = []
srList = self.searcher.searchForArtistReleases(artist, [])
if not srList:
self.logger.info("Releases For Artist [" + printableArtistName +
"] Not Found")
return None
if srList:
for sr in srList:
title = sr.title
release = self._createDatabaseModelFromSearchModel(artist, title, sr)
self.session.add(release)
releases.append(release)
self.session.commit()
return releases
def get(self, artist, title, doFindIfNotInDB=True, forceRefresh=False):
"""
Query Database for a release with the given title, if not found search and if found save and return results
:param forceRefresh: bool
:param doFindIfNotInDB: bool
:rtype : Release
:param artist: Artist
:param title: str
"""
try:
if not title or not artist:
return None
startTime = arrow.utcnow().datetime
printableTitle = title.encode('ascii', 'ignore').decode('utf-8')
printableArtistName = artist.name.encode('ascii', 'ignore').decode('utf-8')
release = self._getFromDatabaseByTitle(artist, title)
if not release and doFindIfNotInDB or forceRefresh:
if not release:
self.logger.info("Release For Artist [" + printableArtistName +
"] Not Found By Title [" + printableTitle + "]")
else:
self.logger.info("Refreshing Release [" + printableTitle + "] For Artist [" + printableArtistName)
release = Release()
artistReleaseImages = self.session.query(Image) \
.add_column(Image.signature) \
.join(Release) \
.filter(Release.artistId == artist.id).all()
srList = self.searcher.searchForArtistReleases(artist, artistReleaseImages, title)
if not srList:
self.logger.info("Release For Artist [" + printableArtistName +
"] Not Found By Title [" + printableTitle + "]")
return None
sr = srList[0]
if sr:
release = self._createDatabaseModelFromSearchModel(artist, title, sr)
self.session.add(release)
self.session.commit()
elapsedTime = arrow.utcnow().datetime - startTime
self.logger.info(": ReleaseFactory get elapsed time [" + str(elapsedTime) + "]")
return release
except:
self.logger.exception("ReleaseFactory: Error In get()")
pass
return None
def _createDatabaseModelFromSearchModel(self, artist, title, sr):
"""
Take the given SearchResult Release Model and create a Database Model
:type artist: Artist
:type title: str
:type sr: searchEngines.models.Release.Release
"""
createDattabaseModelFromSearchModelRelease = Release()
printableTitle = title.encode('ascii', 'ignore').decode('utf-8')
releaseByExternalIds = self._getFromDatabaseByExternalIds(sr.musicBrainzId,
sr.iTunesId,
sr.lastFMId,
sr.amgId,
sr.spotifyId)
if releaseByExternalIds:
if not releaseByExternalIds.alternateNames:
releaseByExternalIds.alternateNames = []
if title not in releaseByExternalIds.alternateNames:
self.logger.debug("Found Title By External Ids [" +
releaseByExternalIds.title.encode('ascii', 'ignore')
.decode('utf-8') + "] Added [" +
printableTitle + "] To AlternateNames")
if not releaseByExternalIds.alternateNames:
releaseByExternalIds.alternateNames = []
releaseByExternalIds.alternateNames.append(title)
releaseByExternalIds.lastUpdated = arrow.utcnow().datetime
self.session.commit()
return releaseByExternalIds
createDattabaseModelFromSearchModelRelease.artist = artist
createDattabaseModelFromSearchModelRelease.roadieId = sr.roadieId
createDattabaseModelFromSearchModelRelease.title = title
createDattabaseModelFromSearchModelRelease.releaseDate = parseDate(sr.releaseDate)
createDattabaseModelFromSearchModelRelease.trackCount = sr.trackCount
createDattabaseModelFromSearchModelRelease.mediaCount = sr.mediaCount
createDattabaseModelFromSearchModelRelease.thumbnail = sr.thumbnail
createDattabaseModelFromSearchModelRelease.profile = sr.profile
if sr.releaseType == SearchReleaseType.Album:
createDattabaseModelFromSearchModelRelease.releaseType = 'Album'
elif sr.releaseType == SearchReleaseType.EP:
createDattabaseModelFromSearchModelRelease.releaseType = 'EP'
elif sr.releaseType == SearchReleaseType.Single:
createDattabaseModelFromSearchModelRelease.releaseType = 'Single'
createDattabaseModelFromSearchModelRelease.iTunesId = sr.iTunesId
createDattabaseModelFromSearchModelRelease.amgId = sr.amgId
createDattabaseModelFromSearchModelRelease.lastFMId = sr.lastFMId
createDattabaseModelFromSearchModelRelease.lastFMSummary = sr.lastFMSummary
createDattabaseModelFromSearchModelRelease.musicBrainzId = sr.musicBrainzId
createDattabaseModelFromSearchModelRelease.spotifyId = sr.spotifyId
createDattabaseModelFromSearchModelRelease.amgId = sr.amgId
createDattabaseModelFromSearchModelRelease.tags = sr.tags
createDattabaseModelFromSearchModelRelease.alternateNames = sr.alternateNames
createDattabaseModelFromSearchModelRelease.urls = sr.urls
if sr.images:
createDattabaseModelFromSearchModelReleaseimages = []
for image in sr.images:
if image.image:
i = Image()
i.roadieId = image.roadieId
i.url = image.url
i.caption = image.caption
i.image = image.image
i.signature = image.signature
createDattabaseModelFromSearchModelReleaseimages.append(i)
createDattabaseModelFromSearchModelRelease.images = createDattabaseModelFromSearchModelReleaseimages
self.logger.debug(
"= Added [" + str(len(createDattabaseModelFromSearchModelRelease.images)) + "] Images to Release")
# TODO
# See if cover file found in Release Folder
# coverFile = os.path.join(mp3Folder, "cover.jpg")
# if os.path.isfile(coverFile):
# ba = self.readImageThumbnailBytesFromFile(coverFile)
# else:
# coverFile = os.path.join(mp3Folder, "front.jpg")
# if os.path.isfile(coverFile):
# ba = self.readImageThumbnailBytesFromFile(coverFile)
# # if no bytes found see if MusicBrainz has cover art
# if not ba:
# coverArtBytes = mb.lookupCoverArt(release.MusicBrainzId)
# if coverArtBytes:
# try:
# img = Image.open(io.BytesIO(coverArtBytes))
# img.thumbnail(self.thumbnailSize)
# b = io.BytesIO()
# img.save(b, "JPEG")
# ba = b.getvalue()
# except:
# pass
if sr.genres:
createDattabaseModelFromSearchModelRelease.genres = []
for genre in sr.genres:
dbGenre = self.session.query(Genre).filter(Genre.name == genre.name).first()
if not dbGenre:
g = Genre()
g.name = genre.name
g.roadieId = genre.roadieId
createDattabaseModelFromSearchModelRelease.genres.append(g)
else:
createDattabaseModelFromSearchModelRelease.genres.append(dbGenre)
if sr.releaseLabels:
createDattabaseModelFromSearchModelRelease.releaseLabels = []
for srReleaseLabel in sr.releaseLabels:
l = self._getLabelFromDatabase(srReleaseLabel.label.name)
if not l:
l = Label()
l.roadieId = srReleaseLabel.label.roadieId
l.musicBrainzId = srReleaseLabel.label.musicBrainzId
l.beginDate = srReleaseLabel.label.beginDate
l.end = srReleaseLabel.label.endDate
l.imageUrl = srReleaseLabel.label.imageUrl
l.tags = srReleaseLabel.label.tags
if srReleaseLabel.label.alternateNames:
srLabelAlternateNames = []
for srLabelAn in srReleaseLabel.label.alternateNames:
srLabelAlternateNames.append(srLabelAn.replace("|", ","))
l.alternateNames = srLabelAlternateNames
l.sortName = srReleaseLabel.label.sortName
l.name = srReleaseLabel.label.name
if l:
rl = ReleaseLabel()
rl.roadieId = srReleaseLabel.roadieId
rl.catalogNumber = srReleaseLabel.catalogNumber
rl.beginDate = parseDate(srReleaseLabel.beginDate)
rl.endDate = parseDate(srReleaseLabel.endDate)
rl.label = l
if rl not in createDattabaseModelFromSearchModelRelease.releaseLabels:
createDattabaseModelFromSearchModelRelease.releaseLabels.append(rl)
if sr.media:
createDattabaseModelFromSearchModelRelease.media = []
for srMedia in sr.media:
media = ReleaseMedia()
media.roadieId = srMedia.roadieId
media.releaseMediaNumber = int(srMedia.releaseMediaNumber)
# The first media is release 1 not release 0
if media.releaseMediaNumber < 1:
media.releaseMediaNumber = 1
media.releaseSubTitle = srMedia.releaseSubTitle
media.trackCount = srMedia.trackCount
if srMedia.tracks:
media.tracks = []
for srTrack in srMedia.tracks:
track = Track()
track.roadieId = srTrack.roadieId
track.partTitles = srTrack.partTitles
track.musicBrainzId = srTrack.musicBrainzId
track.amgId = srTrack.amgId
track.spotifyId = srTrack.spotifyId
track.title = srTrack.title
track.trackNumber = srTrack.trackNumber
track.duration = srTrack.duration
track.tags = srTrack.tags
track.alternateNames = []
cleanedTitle = createCleanedName(srTrack.title)
if cleanedTitle != srTrack.title.lower().strip():
track.alternateNames.append(cleanedTitle)
media.tracks.append(track)
createDattabaseModelFromSearchModelRelease.media.append(media)
createDattabaseModelFromSearchModelRelease.mediaCount = len(
createDattabaseModelFromSearchModelRelease.media)
return createDattabaseModelFromSearchModelRelease
def _getAllFromDatabaseForArtist(self, artist):
if not artist:
return None
return self.session.query(Release).filter(Release.artistId == artist.id).order_by(Release.releaseDate).all()
def _getFromDatabaseByTitle(self, artist, title):
if not title:
return None
title = title.lower().strip()
cleanedTitle = createCleanedName(title)
stmt = or_(func.lower(Release.title) == title,
text("(lower(alternateNames) = '" + title.replace("'", "''") + "'" + ""
" OR alternateNames like '" + title.replace(
"'", "''") + "|%'" +
" OR alternateNames like '%|" + title.replace("'", "''") + "|%'" +
" OR alternateNames like '%|" + title.replace("'", "''") + "')"),
text("(alternateNames = '" + cleanedTitle + "'" + ""
" OR alternateNames like '" + cleanedTitle + "|%'" +
" OR alternateNames like '%|" + cleanedTitle + "|%'" +
" OR alternateNames like '%|" + cleanedTitle + "')")
)
return self.session.query(Release).filter(Release.artistId == artist.id).filter(stmt).first()
def _getLabelFromDatabase(self, name):
if not name:
return None
name = name.lower().strip()
stmt = or_(func.lower(Label.name) == name,
text("(lower(alternateNames) = '" + name.replace("'", "''") + "'" + ""
" OR alternateNames like '" + name.replace(
"'", "''") + "|%'" +
" OR alternateNames like '%|" + name.replace("'", "''") + "|%'" +
" OR alternateNames like '%|" + name.replace("'", "''") + "')"))
return self.session.query(Label).filter(stmt).first()
def _getFromDatabaseByExternalIds(self, musicBrainzId, iTunesId, lastFMId, amgId, spotifyId):
mb = and_(Release.musicBrainzId == musicBrainzId, musicBrainzId is not None)
it = and_(Release.iTunesId == iTunesId, iTunesId is not None)
lf = and_(Release.lastFMId == lastFMId, lastFMId is not None)
ag = and_(Release.amgId == amgId, amgId is not None)
sp = and_(Release.spotifyId == spotifyId, spotifyId is not None)
stmt = or_(mb, it, lf, ag, sp)
return self.session.query(Release).filter(stmt).first()
def _getFromDatabaseByRoadieId(self, roadieId):
return self.session.query(Release).filter(Release.roadieId == roadieId).first()
def create(self, artist, title, trackCount, releaseDate):
if not artist or not title or not trackCount or not releaseDate:
return None
release = Release()
release.title = title
release.releaseDate = parseDate(releaseDate)
release.trackCount = trackCount
release.artistId = artist.id
release.createdDate = arrow.utcnow().datetime
release.roadieId = str(uuid.uuid4())
release.alternateNames = []
cleanedTitle = createCleanedName(title)
if cleanedTitle != title.lower().strip():
release.alternateNames.append(cleanedTitle)
return release
def add(self, release):
self.session.add(release)
self.session.commit()
def delete(self, release, pathToTrack, deleteFiles=False):
"""
Performs all necessary steps to delete a Release and optionally Release Tracks
:param pathToTrack: Method to generate Full Path for Release Media Tracks
:param release: Releasesaasdf
:type deleteFiles: bool
"""
if not release:
return False
try:
if deleteFiles:
try:
for deleteReleaseMedia in release.media:
for track in deleteReleaseMedia.tracks:
trackPath = pathToTrack(track)
trackFolder = os.path.dirname(trackPath)
os.remove(trackPath)
# if the folder is empty then delete the folder as well
if trackFolder:
if not os.listdir(trackFolder):
os.rmdir(trackFolder)
except OSError:
pass
release.genres = []
self.session.commit()
self.session.delete(release)
self.session.commit()
return True
except:
self.session.rollback()
self.logger.exception("Error Deleting Release")
return False
class ArtistSearcher(object):
"""
Query Enabled Search Engines and Find Artist Information and aggregate results.
"""
allMusicSearcher = None
spotifySearcher = None
mbSearcher = None
lastFMSearcher = None
imageSearcher = None
iTunesSearcher = None
imageSearcher = None
artistThumbnailSize = 160, 160
releaseThumbnailSize = 80, 80
imageMaximumSize = 500, 500
cache = dict()
imageCache = dict()
def __init__(self, referer=None):
self.referer = referer
if not self.referer or self.referer.startswith("http://localhost"):
self.referer = "http://github.com/sphildreth/roadie"
self.logger = Logger()
self.allMusicSearcher = AllMusicGuide(self.referer)
self.spotifySearcher = Spotify(self.referer)
self.mbSearcher = MusicBrainz(self.referer)
self.lastFMSearcher = LastFM(self.referer)
self.imageSearcher = ImageSearcher()
self.iTunesSearcher = iTunes(self.referer)
self.imageSearcher = ImageSearcher(self.referer)
def searchForArtist(self, name):
"""
Perform a search in all enabled search engines and return an aggregate Artist for the given Artist name
:param name: String
Name of the Artist to find
:return: Artist
Populated Artist or None if error or not found
"""
if not name:
return None
if name in self.cache:
return self.cache[name]
try:
startTime = arrow.utcnow().datetime
artist = Artist(name=name)
artist.roadieId = str(uuid.uuid4())
if self.iTunesSearcher.IsActive:
artist = artist.mergeWithArtist(self.iTunesSearcher.lookupArtist(name))
if self.mbSearcher.IsActive:
artist = artist.mergeWithArtist(self.mbSearcher.lookupArtist(name))
if self.lastFMSearcher.IsActive:
artist = artist.mergeWithArtist(self.lastFMSearcher.lookupArtist(name))
if self.spotifySearcher.IsActive:
artist = artist.mergeWithArtist(self.spotifySearcher.lookupArtist(name))
if self.allMusicSearcher.IsActive:
artist = artist.mergeWithArtist(self.allMusicSearcher.lookupArtist(name))
if artist:
# Fetch images with only urls, remove any with neither URL or BLOB
if artist.images:
images = []
firstImageInImages = None
for image in artist.images:
if not image.image and image.url:
image.image = self.imageSearcher.getImageBytesForUrl(image.url)
if image.image:
# Resize to maximum image size and convert to JPEG
img = Image.open(io.BytesIO(image.image)).convert('RGB')
img.resize(self.imageMaximumSize)
b = io.BytesIO()
img.save(b, "JPEG")
image.image = b.getvalue()
firstImageInImages = firstImageInImages or image.image
image.signature = image.averageHash()
images.append(image)
if images:
dedupedImages = []
imageSignatures = []
for image in images:
if image.signature not in imageSignatures:
imageSignatures.append(image.signature)
dedupedImages.append(image)
artist.images = dedupedImages
if not artist.thumbnail and firstImageInImages:
try:
img = Image.open(io.BytesIO(firstImageInImages)).convert('RGB')
img.thumbnail(self.artistThumbnailSize)
b = io.BytesIO()
img.save(b, "JPEG")
artist.thumbnail = b.getvalue()
except:
pass
# Add special search names to alternate names
if not artist.alternateNames:
artist.alternateNames = []
if artist.name not in artist.alternateNames:
cleanedArtistName = createCleanedName(artist.name)
if cleanedArtistName != artist.name.lower().strip() and \
cleanedArtistName not in artist.alternateNames:
artist.alternateNames.append(cleanedArtistName)
if not artist.bioContext:
try:
artist.bioContext = wikipedia.summary(artist.name)
except:
pass
self.cache[name] = artist
elapsedTime = arrow.utcnow().datetime - startTime
printableName = name.encode('ascii', 'ignore').decode('utf-8')
self.logger.debug("searchForArtist Elapsed Time [" + str(elapsedTime) + "] Name [" + printableName +
"] Found [" + (artist.name if artist else "") +
"] MusicBrainzId [" + str(artist.musicBrainzId) + "] " +
" iTunesId [" + str(artist.iTunesId) + "] " +
" amgId [" + str(artist.amgId) + "]" +
" spotifyId [" + str(artist.spotifyId) + "]"
.encode('ascii', 'ignore').decode('utf-8') + "]")
return artist
except:
self.logger.exception("Error In searchForArtist")
return None
def _mergeReleaseLists(self, left, right):
if left and not right:
return left
elif not left and right:
return right
elif not left and not right:
return []
else:
mergeReleaseListsStart = arrow.utcnow()
mergedReleases = left
# Merge the right to the result
for rRelease in right:
foundRightInMerged = False
for mRelease in mergedReleases:
if mRelease == rRelease:
mRelease.mergeWithRelease(rRelease)
foundRightInMerged = True
break
if not foundRightInMerged:
mergedReleases.append(rRelease)
mergedReleaseElapsed = arrow.utcnow() - mergeReleaseListsStart
self.logger.debug("= MergeReleaseLists left size [" + str(len(left)) + "], right size [" + str(
len(right)) + "] Elapsed Time [" + str(mergedReleaseElapsed) + "]")
return mergedReleases
def searchForArtistReleases(self, artist, artistReleaseImages, titleFilter=None):
"""
Using the given populated Artist find all releases, with an optional filter
:param artist: Artist
Artist to find releases for
:param artistReleaseImages: list
Collection if image signatures for Artist for deduping
:param titleFilter: String
Optional filter of release Title to only include in results
:return: iterable Release
Collection of releases found for artist
"""
if not artist:
return None
try:
startTime = arrow.utcnow().datetime
releases = []
if self.iTunesSearcher.IsActive:
releases = self._mergeReleaseLists(releases, self.iTunesSearcher.searchForRelease(artist, titleFilter))
if self.mbSearcher.IsActive:
releases = self._mergeReleaseLists(releases, self.mbSearcher.searchForRelease(artist, titleFilter))
if self.lastFMSearcher.IsActive and releases:
mbIdList = []
if not titleFilter:
mbIdList = [x.musicBrainzId for x in releases if x.musicBrainzId]
else:
for x in releases:
if isEqual(x.title, titleFilter):
mbIdList.append(x.musicBrainzId)
break
if mbIdList:
releases = self._mergeReleaseLists(releases,
self.lastFMSearcher.lookupReleasesForMusicBrainzIdList(artist,
mbIdList))
if self.spotifySearcher.IsActive:
releases = self._mergeReleaseLists(releases, self.spotifySearcher.searchForRelease(artist, titleFilter))
if releases:
self.logger.debug(
"searchForArtistReleases Found [" + str(len(releases)) + "] For title [" + str(titleFilter) + "]")
for searchForArtistRelease in releases:
if searchForArtistRelease.coverUrl:
coverImage = ArtistImage(searchForArtistRelease.coverUrl)
searchForArtistRelease.images.append(coverImage)
# Fetch images with only urls, remove any with neither URL or BLOB
if searchForArtistRelease.images:
images = []
for image in searchForArtistRelease.images:
if not image.image and image.url:
image.image = self.getImageForUrl(image.url)
if image.image:
# Resize to maximum image size and convert to JPEG
img = Image.open(io.BytesIO(image.image)).convert('RGB')
img.resize(self.imageMaximumSize)
b = io.BytesIO()
img.save(b, "JPEG")
image.image = b.getvalue()
# Hash image for deduping
image.signature = image.averageHash()
if image.signature:
images.append(image)
if not images:
searchForArtistRelease.images = []
else:
dedupedImages = []
imageSignatures = artistReleaseImages or []
for image in images:
if image.signature not in imageSignatures:
imageSignatures.append(image.signature)
dedupedImages.append(image)
searchForArtistRelease.images = dedupedImages
if not searchForArtistRelease.thumbnail:
try:
firstImageInImages = None
for image in searchForArtistRelease.images:
firstImageInImages = firstImageInImages or image.image
if firstImageInImages:
break
img = Image.open(io.BytesIO(firstImageInImages)).convert('RGB')
img.thumbnail(self.releaseThumbnailSize)
b = io.BytesIO()
img.save(b, "JPEG")
searchForArtistRelease.thumbnail = b.getvalue()
except:
pass
if titleFilter and releases:
filteredReleases = []
cleanedTitleFilter = createCleanedName(titleFilter)
for searchForArtistRelease in releases:
if isEqual(searchForArtistRelease.title,
titleFilter) or cleanedTitleFilter in searchForArtistRelease.alternateNames:
filteredReleases.append(searchForArtistRelease)
releases = filteredReleases
elapsedTime = arrow.utcnow().datetime - startTime
self.logger.debug("searchForArtistReleases ElapseTime [" + str(elapsedTime) + "]")
return releases
except:
self.logger.exception("Error In searchForArtistReleases")
pass
return None
def getImageForUrl(self, url):
if url not in self.imageCache:
self.imageCache[url] = self.imageSearcher.getImageBytesForUrl(url)
self.logger.debug("= Downloading Image [" + str(url) + "]")
return self.imageCache[url]
