def addArtisttoDB(artistid, extrasonly=False, forcefull=False):
# Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
from headphones import cache
# Can't add various artists - throws an error from MB
if artistid in blacklisted_special_artists:
logger.warn('Cannot import blocked special purpose artist with id' + artistid)
return
# We'll use this to see if we should update the 'LastUpdated' time stamp
errors = False
myDB = db.DBConnection()
# Delete from blacklist if it's on there
myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ArtistID": artistid}
# Don't replace a known artist name with an "Artist ID" placeholder
dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone()
# Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
if not dbartist:
newValueDict = {"ArtistName": "Artist ID: %s" % (artistid),
"Status": "Loading",
"IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS,
"Extras": headphones.CONFIG.EXTRAS}
else:
newValueDict = {"Status": "Loading"}
myDB.upsert("artists", newValueDict, controlValueDict)
artist = mb.getArtist(artistid, extrasonly)
if artist and artist.get('artist_name') in blacklisted_special_artist_names:
logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name'))
myDB.action('DELETE from artists WHERE ArtistID=?', [artistid])
#in case it's already in the db
myDB.action('DELETE from albums WHERE ArtistID=?', [artistid])
myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid])
return
if not artist:
logger.warn("Error fetching artist info. ID: " + artistid)
if dbartist is None:
newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid),
"Status": "Active"}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("artists", newValueDict, controlValueDict)
return
if artist['artist_name'].startswith('The '):
sortname = artist['artist_name'][4:]
else:
sortname = artist['artist_name']
logger.info(u"Now adding/updating: " + artist['artist_name'])
controlValueDict = {"ArtistID": artistid}
newValueDict = {"ArtistName": artist['artist_name'],
"ArtistSortName": sortname,
"DateAdded": helpers.today(),
"Status": "Loading"}
myDB.upsert("artists", newValueDict, controlValueDict)
# See if we need to grab extras. Artist specific extras take precedence
# over global option. Global options are set when adding a new artist
try:
db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone()
includeExtras = db_artist['IncludeExtras']
except IndexError:
includeExtras = False
# Clean all references to release group in dB that are no longer referenced
# from the musicbrainz refresh
group_list = []
force_repackage = 0
# Don't nuke the database if there's a MusicBrainz error
if len(artist['releasegroups']) != 0:
for groups in artist['releasegroups']:
group_list.append(groups['id'])
if not extrasonly:
remove_missing_groups_from_albums = myDB.select("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
else:
remove_missing_groups_from_albums = myDB.select('SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid])
for items in remove_missing_groups_from_albums:
if items['AlbumID'] not in group_list:
# Remove all from albums/tracks that aren't in release groups
myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']])
myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']])
myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']])
myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']])
myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']])
logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (artist['artist_name'], items['AlbumID']))
if not extrasonly:
force_repackage = 1
else:
if not extrasonly:
logger.info("[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name'])
# Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
album_searches = []
for rg in artist['releasegroups']:
al_title = rg['title']
today = helpers.today()
rgid = rg['id']
skip_log = 0
#Make a user configurable variable to skip update of albums with release dates older than this date (in days)
pause_delta = headphones.CONFIG.MB_IGNORE_AGE
rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
if not forcefull:
new_release_group = False
try:
check_release_date = rg_exists['ReleaseDate']
except TypeError:
check_release_date = None
new_release_group = True
if new_release_group:
logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title']))
new_releases = mb.get_new_releases(rgid, includeExtras)
else:
if check_release_date is None or check_release_date == u"None":
logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
new_releases = mb.get_new_releases(rgid, includeExtras, True)
else:
if len(check_release_date) == 10:
release_date = check_release_date
elif len(check_release_date) == 7:
release_date = check_release_date + "-31"
elif len(check_release_date) == 4:
release_date = check_release_date + "-12-31"
else:
release_date = today
if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
logger.info("[%s] Now updating: %s (Release Date <%s Days)", artist['artist_name'], rg['title'], pause_delta)
new_releases = mb.get_new_releases(rgid, includeExtras, True)
else:
logger.info("[%s] Skipping: %s (Release Date >%s Days)", artist['artist_name'], rg['title'], pause_delta)
skip_log = 1
new_releases = 0
if force_repackage == 1:
new_releases = -1
logger.info('[%s] Forcing repackage of %s (Release Group Removed)', artist['artist_name'], al_title)
else:
new_releases = new_releases
else:
logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'], rg['title'])
new_releases = mb.get_new_releases(rgid, includeExtras, forcefull)
if new_releases != 0:
# Dump existing hybrid release since we're repackaging/replacing it
myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']])
myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']])
myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']])
myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']])
myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rg['id']])
# This will be used later to build a hybrid release
fullreleaselist = []
# Search for releases within a release group
find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']])
# Build the dictionary for the fullreleaselist
for items in find_hybrid_releases:
if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing
hybrid_release_id = items['ReleaseID']
newValueDict = {"ArtistID": items['ArtistID'],
"ArtistName": items['ArtistName'],
"AlbumTitle": items['AlbumTitle'],
"AlbumID": items['AlbumID'],
"AlbumASIN": items['AlbumASIN'],
"ReleaseDate": items['ReleaseDate'],
"Type": items['Type'],
"ReleaseCountry": items['ReleaseCountry'],
"ReleaseFormat": items['ReleaseFormat']
}
find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id])
totalTracks = 1
hybrid_track_array = []
for hybrid_tracks in find_hybrid_tracks:
hybrid_track_array.append({
'number': hybrid_tracks['TrackNumber'],
'title': hybrid_tracks['TrackTitle'],
'id': hybrid_tracks['TrackID'],
#'url': hybrid_tracks['TrackURL'],
'duration': hybrid_tracks['TrackDuration']
})
totalTracks += 1
newValueDict['ReleaseID'] = hybrid_release_id
newValueDict['Tracks'] = hybrid_track_array
fullreleaselist.append(newValueDict)
# Basically just do the same thing again for the hybrid release
# This may end up being called with an empty fullreleaselist
try:
hybridrelease = getHybridRelease(fullreleaselist)
logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title']))
except Exception as e:
errors = True
logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'], rg['title'], e))
continue
# Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
# We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
# The hybrid won't have a country or a format
controlValueDict = {"ReleaseID": rg['id']}
newValueDict = {"ArtistID": artistid,
"ArtistName": artist['artist_name'],
"AlbumTitle": rg['title'],
"AlbumID": rg['id'],
"AlbumASIN": hybridrelease['AlbumASIN'],
"ReleaseDate": hybridrelease['ReleaseDate'],
"Type": rg['type']
}
myDB.upsert("allalbums", newValueDict, controlValueDict)
for track in hybridrelease['Tracks']:
cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
controlValueDict = {"TrackID": track['id'],
"ReleaseID": rg['id']}
newValueDict = {"ArtistID": artistid,
"ArtistName": artist['artist_name'],
"AlbumTitle": rg['title'],
"AlbumASIN": hybridrelease['AlbumASIN'],
"AlbumID": rg['id'],
"TrackTitle": track['title'],
"TrackDuration": track['duration'],
"TrackNumber": track['number'],
"CleanName": cleanname
}
match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
if not match:
match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
#if not match:
#match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
if match:
newValueDict['Location'] = match['Location']
newValueDict['BitRate'] = match['BitRate']
newValueDict['Format'] = match['Format']
#myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location']))
myDB.upsert("alltracks", newValueDict, controlValueDict)
# Delete matched tracks from the have table
#myDB.action('DELETE from have WHERE Matched="True"')
# If there's no release in the main albums tables, add the default (hybrid)
# If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
# check if the album already exists
if not rg_exists:
releaseid = rg['id']
else:
releaseid = rg_exists['ReleaseID']
if not releaseid:
releaseid = rg['id']
album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone()
controlValueDict = {"AlbumID": rg['id']}
newValueDict = {"ArtistID": album['ArtistID'],
"ArtistName": album['ArtistName'],
"AlbumTitle": album['AlbumTitle'],
"ReleaseID": album['ReleaseID'],
"AlbumASIN": album['AlbumASIN'],
"ReleaseDate": album['ReleaseDate'],
"Type": album['Type'],
"ReleaseCountry": album['ReleaseCountry'],
"ReleaseFormat": album['ReleaseFormat']
}
if rg_exists:
newValueDict['DateAdded'] = rg_exists['DateAdded']
newValueDict['Status'] = rg_exists['Status']
else:
today = helpers.today()
newValueDict['DateAdded'] = today
if headphones.CONFIG.AUTOWANT_ALL:
newValueDict['Status'] = "Wanted"
elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING:
newValueDict['Status'] = "Wanted"
# Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
# The first test just makes sure we have year-month-day
elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
newValueDict['Status'] = "Wanted"
else:
newValueDict['Status'] = "Skipped"
myDB.upsert("albums", newValueDict, controlValueDict)
tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()
# This is used to see how many tracks you have from an album - to
# mark it as downloaded. Default is 80%, can be set in config as
# ALBUM_COMPLETION_PCT
total_track_count = len(tracks)
if total_track_count == 0:
logger.warning("Total track count is zero for Release ID " +
"'%s', skipping.", releaseid)
continue
for track in tracks:
controlValueDict = {"TrackID": track['TrackID'],
"AlbumID": rg['id']}
newValueDict = {"ArtistID": track['ArtistID'],
"ArtistName": track['ArtistName'],
"AlbumTitle": track['AlbumTitle'],
"AlbumASIN": track['AlbumASIN'],
"ReleaseID": track['ReleaseID'],
"TrackTitle": track['TrackTitle'],
"TrackDuration": track['TrackDuration'],
"TrackNumber": track['TrackNumber'],
"CleanName": track['CleanName'],
"Location": track['Location'],
"Format": track['Format'],
"BitRate": track['BitRate']
}
myDB.upsert("tracks", newValueDict, controlValueDict)
# Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']]))
marked_as_downloaded = False
if rg_exists:
if rg_exists['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
marked_as_downloaded = True
else:
if ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
marked_as_downloaded = True
logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
cache.getThumb(AlbumID=rg['id'])
# Start a search for the album if it's new, hasn't been marked as
# downloaded and autowant_all is selected. This search is deferred,
# in case the search failes and the rest of the import will halt.
if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL:
album_searches.append(rg['id'])
else:
if skip_log == 0:
logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title']))
time.sleep(3)
finalize_update(artistid, artist['artist_name'], errors)
logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
cache.getThumb(ArtistID=artistid)
if errors:
logger.info("[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name']))
else:
myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']])
logger.info(u"Updating complete for: %s" % artist['artist_name'])
# Start searching for newly added albums
if album_searches:
from headphones import searcher
logger.info("Start searching for %d albums.", len(album_searches))
for album_search in album_searches:
searcher.searchforalbum(albumid=album_search)
def addArtisttoDB(artistid, extrasonly=False, forcefull=False):
# Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
from headphones import cache
# Can't add various artists - throws an error from MB
if artistid in blacklisted_special_artists:
logger.warn("Cannot import blocked special purpose artist with id" + artistid)
return
# We'll use this to see if we should update the 'LastUpdated' time stamp
errors = False
myDB = db.DBConnection()
# Delete from blacklist if it's on there
myDB.action("DELETE from blacklist WHERE ArtistID=?", [artistid])
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ArtistID": artistid}
# Don't replace a known artist name with an "Artist ID" placeholder
dbartist = myDB.action("SELECT * FROM artists WHERE ArtistID=?", [artistid]).fetchone()
# Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
if not dbartist:
newValueDict = {
"ArtistName": "Artist ID: %s" % (artistid),
"Status": "Loading",
"IncludeExtras": headphones.INCLUDE_EXTRAS,
"Extras": headphones.EXTRAS,
}
else:
newValueDict = {"Status": "Loading"}
myDB.upsert("artists", newValueDict, controlValueDict)
artist = mb.getArtist(artistid, extrasonly)
if artist and artist.get("artist_name") in blacklisted_special_artist_names:
logger.warn("Cannot import blocked special purpose artist: %s" % artist.get("artist_name"))
myDB.action("DELETE from artists WHERE ArtistID=?", [artistid])
# in case it's already in the db
myDB.action("DELETE from albums WHERE ArtistID=?", [artistid])
myDB.action("DELETE from tracks WHERE ArtistID=?", [artistid])
return
if not artist:
logger.warn("Error fetching artist info. ID: " + artistid)
if dbartist is None:
newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("artists", newValueDict, controlValueDict)
return
if artist["artist_name"].startswith("The "):
sortname = artist["artist_name"][4:]
else:
sortname = artist["artist_name"]
logger.info(u"Now adding/updating: " + artist["artist_name"])
controlValueDict = {"ArtistID": artistid}
newValueDict = {
"ArtistName": artist["artist_name"],
"ArtistSortName": sortname,
"DateAdded": helpers.today(),
"Status": "Loading",
}
myDB.upsert("artists", newValueDict, controlValueDict)
# See if we need to grab extras. Artist specific extras take precedence over global option
# Global options are set when adding a new artist
myDB = db.DBConnection()
try:
db_artist = myDB.action("SELECT IncludeExtras, Extras from artists WHERE ArtistID=?", [artistid]).fetchone()
includeExtras = db_artist["IncludeExtras"]
except IndexError:
includeExtras = False
# Clean all references to release group in dB that are no longer referenced in musicbrainz
group_list = []
force_repackage = 0
# Don't nuke the database if there's a MusicBrainz error
if len(artist["releasegroups"]) != 0 and not extrasonly:
for groups in artist["releasegroups"]:
group_list.append(groups["id"])
remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
for items in remove_missing_groups_from_albums:
if items["AlbumID"] not in group_list:
# Remove all from albums/tracks that aren't in release groups
myDB.action("DELETE FROM albums WHERE AlbumID=?", [items["AlbumID"]])
myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items["AlbumID"]])
myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items["AlbumID"]])
myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items["AlbumID"]])
logger.info(
"[%s] Removing all references to release group %s to reflect MusicBrainz"
% (artist["artist_name"], items["AlbumID"])
)
force_repackage = 1
else:
logger.info("[%s] Error pulling data from MusicBrainz: Maintaining dB" % artist["artist_name"])
# Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
for rg in artist["releasegroups"]:
al_title = rg["title"]
today = helpers.today()
rgid = rg["id"]
skip_log = 0
# Make a user configurable variable to skip update of albums with release dates older than this date (in days)
pause_delta = headphones.MB_IGNORE_AGE
check_release_date = myDB.action(
"SELECT ReleaseDate, Status from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)
).fetchone()
# Skip update if Status set
if check_release_date and check_release_date[1]:
logger.info(
"[%s] Not updating: %s (Status is %s, skipping)"
% (artist["artist_name"], rg["title"], check_release_date[1])
)
continue
if not forcefull:
if check_release_date:
if check_release_date[0] is None:
logger.info("[%s] Now updating: %s (No Release Date)" % (artist["artist_name"], rg["title"]))
new_releases = mb.get_new_releases(rgid, includeExtras, True)
else:
if len(check_release_date[0]) == 10:
release_date = check_release_date[0]
elif len(check_release_date[0]) == 7:
release_date = check_release_date[0] + "-31"
elif len(check_release_date[0]) == 4:
release_date = check_release_date[0] + "-12-31"
else:
release_date = today
if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
logger.info(
"[%s] Now updating: %s (Release Date <%s Days) "
% (artist["artist_name"], rg["title"], pause_delta)
)
new_releases = mb.get_new_releases(rgid, includeExtras, True)
else:
logger.info(
"[%s] Skipping: %s (Release Date >%s Days)"
% (artist["artist_name"], rg["title"], pause_delta)
)
skip_log = 1
new_releases = 0
else:
logger.info("[%s] Now adding: %s (New Release Group)" % (artist["artist_name"], rg["title"]))
new_releases = mb.get_new_releases(rgid, includeExtras)
if force_repackage == 1:
new_releases = -1
logger.info("[%s] Forcing repackage of %s (Release Group Removed)" % (artist["artist_name"], al_title))
else:
new_releases = new_releases
else:
logger.info("[%s] Now adding/updating: %s (Comprehensive Force)" % (artist["artist_name"], rg["title"]))
new_releases = mb.get_new_releases(rgid, includeExtras, forcefull)
# What this does is adds new releases per artist to the allalbums + alltracks databases
# new_releases = mb.get_new_releases(rgid,includeExtras)
# print al_title
# print new_releases
if new_releases != 0:
# Dump existing hybrid release since we're repackaging/replacing it
myDB.action("DELETE from albums WHERE ReleaseID=?", [rg["id"]])
myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg["id"]])
myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg["id"]])
myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg["id"]])
# This will be used later to build a hybrid release
fullreleaselist = []
# Search for releases within a release group
find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg["id"]])
# Build the dictionary for the fullreleaselist
for items in find_hybrid_releases:
if (
items["ReleaseID"] != rg["id"]
): # don't include hybrid information, since that's what we're replacing
hybrid_release_id = items["ReleaseID"]
newValueDict = {
"ArtistID": items["ArtistID"],
"ArtistName": items["ArtistName"],
"AlbumTitle": items["AlbumTitle"],
"AlbumID": items["AlbumID"],
"AlbumASIN": items["AlbumASIN"],
"ReleaseDate": items["ReleaseDate"],
"Type": items["Type"],
"ReleaseCountry": items["ReleaseCountry"],
"ReleaseFormat": items["ReleaseFormat"],
}
find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id])
totalTracks = 1
hybrid_track_array = []
for hybrid_tracks in find_hybrid_tracks:
hybrid_track_array.append(
{
"number": hybrid_tracks["TrackNumber"],
"title": hybrid_tracks["TrackTitle"],
"id": hybrid_tracks["TrackID"],
#'url': hybrid_tracks['TrackURL'],
"duration": hybrid_tracks["TrackDuration"],
}
)
totalTracks += 1
newValueDict["ReleaseID"] = hybrid_release_id
newValueDict["Tracks"] = hybrid_track_array
fullreleaselist.append(newValueDict)
# print fullreleaselist
# Basically just do the same thing again for the hybrid release
# This may end up being called with an empty fullreleaselist
try:
hybridrelease = getHybridRelease(fullreleaselist)
logger.info("[%s] Packaging %s releases into hybrid title" % (artist["artist_name"], rg["title"]))
except Exception, e:
errors = True
logger.warn(
"[%s] Unable to get hybrid release information for %s: %s" % (artist["artist_name"], rg["title"], e)
)
continue
# Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
# We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
# The hybrid won't have a country or a format
controlValueDict = {"ReleaseID": rg["id"]}
newValueDict = {
"ArtistID": artistid,
"ArtistName": artist["artist_name"],
"AlbumTitle": rg["title"],
"AlbumID": rg["id"],
"AlbumASIN": hybridrelease["AlbumASIN"],
"ReleaseDate": hybridrelease["ReleaseDate"],
"Type": rg["type"],
}
myDB.upsert("allalbums", newValueDict, controlValueDict)
for track in hybridrelease["Tracks"]:
cleanname = helpers.cleanName(artist["artist_name"] + " " + rg["title"] + " " + track["title"])
controlValueDict = {"TrackID": track["id"], "ReleaseID": rg["id"]}
newValueDict = {
"ArtistID": artistid,
"ArtistName": artist["artist_name"],
"AlbumTitle": rg["title"],
"AlbumASIN": hybridrelease["AlbumASIN"],
"AlbumID": rg["id"],
"TrackTitle": track["title"],
"TrackDuration": track["duration"],
"TrackNumber": track["number"],
"CleanName": cleanname,
}
match = myDB.action(
"SELECT Location, BitRate, Format from have WHERE CleanName=?", [cleanname]
).fetchone()
if not match:
match = myDB.action(
"SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?",
[artist["artist_name"], rg["title"], track["title"]],
).fetchone()
# if not match:
# match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
if match:
newValueDict["Location"] = match["Location"]
newValueDict["BitRate"] = match["BitRate"]
newValueDict["Format"] = match["Format"]
# myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
myDB.action("UPDATE have SET Matched=? WHERE Location=?", (rg["id"], match["Location"]))
myDB.upsert("alltracks", newValueDict, controlValueDict)
# Delete matched tracks from the have table
# myDB.action('DELETE from have WHERE Matched="True"')
# If there's no release in the main albums tables, add the default (hybrid)
# If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
# check if the album already exists
rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg["id"]]).fetchone()
if not rg_exists:
releaseid = rg["id"]
else:
releaseid = rg_exists["ReleaseID"]
album = myDB.action("SELECT * from allalbums WHERE ReleaseID=?", [releaseid]).fetchone()
controlValueDict = {"AlbumID": rg["id"]}
newValueDict = {
"ArtistID": album["ArtistID"],
"ArtistName": album["ArtistName"],
"AlbumTitle": album["AlbumTitle"],
"ReleaseID": album["ReleaseID"],
"AlbumASIN": album["AlbumASIN"],
"ReleaseDate": album["ReleaseDate"],
"Type": album["Type"],
"ReleaseCountry": album["ReleaseCountry"],
"ReleaseFormat": album["ReleaseFormat"],
}
if not rg_exists:
today = helpers.today()
newValueDict["DateAdded"] = today
if headphones.AUTOWANT_ALL:
newValueDict["Status"] = "Wanted"
elif album["ReleaseDate"] > today and headphones.AUTOWANT_UPCOMING:
newValueDict["Status"] = "Wanted"
# Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
# The first test just makes sure we have year-month-day
elif (
helpers.get_age(album["ReleaseDate"])
and helpers.get_age(today) - helpers.get_age(album["ReleaseDate"]) < 21
and headphones.AUTOWANT_UPCOMING
):
newValueDict["Status"] = "Wanted"
else:
newValueDict["Status"] = "Skipped"
myDB.upsert("albums", newValueDict, controlValueDict)
tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [releaseid]).fetchall()
# This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT
total_track_count = len(tracks)
for track in tracks:
controlValueDict = {"TrackID": track["TrackID"], "AlbumID": rg["id"]}
newValueDict = {
"ArtistID": track["ArtistID"],
"ArtistName": track["ArtistName"],
"AlbumTitle": track["AlbumTitle"],
"AlbumASIN": track["AlbumASIN"],
"ReleaseID": track["ReleaseID"],
"TrackTitle": track["TrackTitle"],
"TrackDuration": track["TrackDuration"],
"TrackNumber": track["TrackNumber"],
"CleanName": track["CleanName"],
"Location": track["Location"],
"Format": track["Format"],
"BitRate": track["BitRate"],
}
myDB.upsert("tracks", newValueDict, controlValueDict)
# Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
have_track_count = len(
myDB.select("SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL", [rg["id"]])
)
marked_as_downloaded = False
if rg_exists:
if rg_exists["Status"] == "Skipped" and (
(have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0)
):
myDB.action("UPDATE albums SET Status=? WHERE AlbumID=?", ["Downloaded", rg["id"]])
marked_as_downloaded = True
else:
if (have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0):
myDB.action("UPDATE albums SET Status=? WHERE AlbumID=?", ["Downloaded", rg["id"]])
marked_as_downloaded = True
logger.info(u"[%s] Seeing if we need album art for %s" % (artist["artist_name"], rg["title"]))
cache.getThumb(AlbumID=rg["id"])
# start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected:
if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL:
from headphones import searcher
searcher.searchforalbum(albumid=rg["id"])
else:
if skip_log == 0:
logger.info(u"[%s] No new releases, so no changes made to %s" % (artist["artist_name"], rg["title"]))
def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
# Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
from headphones import cache
# Can't add various artists - throws an error from MB
if artistid in blacklisted_special_artists:
logger.warn('Cannot import blocked special purpose artist with id' +
artistid)
return
# We'll use this to see if we should update the 'LastUpdated' time stamp
errors = False
myDB = db.DBConnection()
# Delete from blacklist if it's on there
myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ArtistID": artistid}
# Don't replace a known artist name with an "Artist ID" placeholder
dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?',
[artistid]).fetchone()
# Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
if not dbartist:
newValueDict = {
"ArtistName": "Artist ID: %s" % (artistid),
"Status": "Loading",
"IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS,
"Extras": headphones.CONFIG.EXTRAS
}
if type == "series":
newValueDict['Type'] = "series"
else:
newValueDict = {"Status": "Loading"}
if dbartist["Type"] == "series":
type = "series"
myDB.upsert("artists", newValueDict, controlValueDict)
if type == "series":
artist = mb.getSeries(artistid)
else:
artist = mb.getArtist(artistid, extrasonly)
if artist and artist.get(
'artist_name') in blacklisted_special_artist_names:
logger.warn('Cannot import blocked special purpose artist: %s' %
artist.get('artist_name'))
myDB.action('DELETE from artists WHERE ArtistID=?', [artistid])
# in case it's already in the db
myDB.action('DELETE from albums WHERE ArtistID=?', [artistid])
myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid])
return
if not artist:
logger.warn("Error fetching artist info. ID: " + artistid)
if dbartist is None:
newValueDict = {
"ArtistName":
"Fetch failed, try refreshing. (%s)" % (artistid),
"Status": "Active"
}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("artists", newValueDict, controlValueDict)
return
if artist['artist_name'].startswith('The '):
sortname = artist['artist_name'][4:]
else:
sortname = artist['artist_name']
logger.info(u"Now adding/updating: " + artist['artist_name'])
controlValueDict = {"ArtistID": artistid}
newValueDict = {
"ArtistName": artist['artist_name'],
"ArtistSortName": sortname,
"DateAdded": helpers.today(),
"Status": "Loading"
}
myDB.upsert("artists", newValueDict, controlValueDict)
# See if we need to grab extras. Artist specific extras take precedence
# over global option. Global options are set when adding a new artist
try:
db_artist = myDB.action(
'SELECT IncludeExtras, Extras from artists WHERE ArtistID=?',
[artistid]).fetchone()
includeExtras = db_artist['IncludeExtras']
except IndexError:
includeExtras = False
# Clean all references to release group in dB that are no longer referenced
# from the musicbrainz refresh
group_list = []
force_repackage = 0
# Don't nuke the database if there's a MusicBrainz error
if len(artist['releasegroups']) != 0:
for groups in artist['releasegroups']:
group_list.append(groups['id'])
if not extrasonly:
remove_missing_groups_from_albums = myDB.select(
"SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
else:
remove_missing_groups_from_albums = myDB.select(
'SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"',
[artistid])
for items in remove_missing_groups_from_albums:
if items['AlbumID'] not in group_list:
# Remove all from albums/tracks that aren't in release groups
myDB.action("DELETE FROM albums WHERE AlbumID=?",
[items['AlbumID']])
myDB.action("DELETE FROM allalbums WHERE AlbumID=?",
[items['AlbumID']])
myDB.action("DELETE FROM tracks WHERE AlbumID=?",
[items['AlbumID']])
myDB.action("DELETE FROM alltracks WHERE AlbumID=?",
[items['AlbumID']])
myDB.action('DELETE from releases WHERE ReleaseGroupID=?',
[items['AlbumID']])
logger.info(
"[%s] Removing all references to release group %s to reflect MusicBrainz refresh"
% (artist['artist_name'], items['AlbumID']))
if not extrasonly:
force_repackage = 1
else:
if not extrasonly:
logger.info(
"[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category"
% artist['artist_name'])
# Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
album_searches = []
for rg in artist['releasegroups']:
al_title = rg['title']
today = helpers.today()
rgid = rg['id']
skip_log = 0
# Make a user configurable variable to skip update of albums with release dates older than this date (in days)
pause_delta = headphones.CONFIG.MB_IGNORE_AGE
rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?",
[rg['id']]).fetchone()
if not forcefull:
new_release_group = False
try:
check_release_date = rg_exists['ReleaseDate']
except TypeError:
check_release_date = None
new_release_group = True
if new_release_group:
logger.info("[%s] Now adding: %s (New Release Group)" %
(artist['artist_name'], rg['title']))
new_releases = mb.get_new_releases(rgid, includeExtras)
else:
if check_release_date is None or check_release_date == u"None":
if headphones.CONFIG.MB_IGNORE_AGE_MISSING is not 1:
logger.info("[%s] Now updating: %s (No Release Date)" %
(artist['artist_name'], rg['title']))
new_releases = mb.get_new_releases(
rgid, includeExtras, True)
else:
logger.info(
"[%s] Skipping update of: %s (No Release Date)" %
(artist['artist_name'], rg['title']))
new_releases = 0
else:
if len(check_release_date) == 10:
release_date = check_release_date
elif len(check_release_date) == 7:
release_date = check_release_date + "-31"
elif len(check_release_date) == 4:
release_date = check_release_date + "-12-31"
else:
release_date = today
if helpers.get_age(today) - helpers.get_age(
release_date) < pause_delta:
logger.info(
"[%s] Now updating: %s (Release Date <%s Days)",
artist['artist_name'], rg['title'], pause_delta)
new_releases = mb.get_new_releases(
rgid, includeExtras, True)
else:
logger.info(
"[%s] Skipping: %s (Release Date >%s Days)",
artist['artist_name'], rg['title'], pause_delta)
skip_log = 1
new_releases = 0
if force_repackage == 1:
new_releases = -1
logger.info(
'[%s] Forcing repackage of %s (Release Group Removed)',
artist['artist_name'], al_title)
else:
new_releases = new_releases
else:
logger.info("[%s] Now adding/updating: %s (Comprehensive Force)",
artist['artist_name'], rg['title'])
new_releases = mb.get_new_releases(rgid, includeExtras, forcefull)
if new_releases != 0:
# Dump existing hybrid release since we're repackaging/replacing it
myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']])
myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']])
myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']])
myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']])
myDB.action('DELETE from releases WHERE ReleaseGroupID=?',
[rg['id']])
# This will be used later to build a hybrid release
fullreleaselist = []
# Search for releases within a release group
find_hybrid_releases = myDB.action(
"SELECT * from allalbums WHERE AlbumID=?", [rg['id']])
# Build the dictionary for the fullreleaselist
for items in find_hybrid_releases:
# don't include hybrid information, since that's what we're replacing
if items['ReleaseID'] != rg['id']:
hybrid_release_id = items['ReleaseID']
newValueDict = {
"ArtistID": items['ArtistID'],
"ArtistName": items['ArtistName'],
"AlbumTitle": items['AlbumTitle'],
"AlbumID": items['AlbumID'],
"AlbumASIN": items['AlbumASIN'],
"ReleaseDate": items['ReleaseDate'],
"Type": items['Type'],
"ReleaseCountry": items['ReleaseCountry'],
"ReleaseFormat": items['ReleaseFormat']
}
find_hybrid_tracks = myDB.action(
"SELECT * from alltracks WHERE ReleaseID=?",
[hybrid_release_id])
totalTracks = 1
hybrid_track_array = []
for hybrid_tracks in find_hybrid_tracks:
hybrid_track_array.append({
'number':
hybrid_tracks['TrackNumber'],
'title':
hybrid_tracks['TrackTitle'],
'id':
hybrid_tracks['TrackID'],
# 'url': hybrid_tracks['TrackURL'],
'duration':
hybrid_tracks['TrackDuration']
})
totalTracks += 1
newValueDict['ReleaseID'] = hybrid_release_id
newValueDict['Tracks'] = hybrid_track_array
fullreleaselist.append(newValueDict)
# Basically just do the same thing again for the hybrid release
# This may end up being called with an empty fullreleaselist
try:
hybridrelease = getHybridRelease(fullreleaselist)
logger.info('[%s] Packaging %s releases into hybrid title' %
(artist['artist_name'], rg['title']))
except Exception as e:
errors = True
logger.warn(
'[%s] Unable to get hybrid release information for %s: %s'
% (artist['artist_name'], rg['title'], e))
continue
# Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
# We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
# The hybrid won't have a country or a format
controlValueDict = {"ReleaseID": rg['id']}
newValueDict = {
"ArtistID": artistid,
"ArtistName": artist['artist_name'],
"AlbumTitle": rg['title'],
"AlbumID": rg['id'],
"AlbumASIN": hybridrelease['AlbumASIN'],
"ReleaseDate": hybridrelease['ReleaseDate'],
"Type": rg['type']
}
myDB.upsert("allalbums", newValueDict, controlValueDict)
for track in hybridrelease['Tracks']:
cleanname = helpers.clean_name(artist['artist_name'] + ' ' +
rg['title'] + ' ' +
track['title'])
controlValueDict = {
"TrackID": track['id'],
"ReleaseID": rg['id']
}
newValueDict = {
"ArtistID": artistid,
"ArtistName": artist['artist_name'],
"AlbumTitle": rg['title'],
"AlbumASIN": hybridrelease['AlbumASIN'],
"AlbumID": rg['id'],
"TrackTitle": track['title'],
"TrackDuration": track['duration'],
"TrackNumber": track['number'],
"CleanName": cleanname
}
match = myDB.action(
'SELECT Location, BitRate, Format from have WHERE CleanName=?',
[cleanname]).fetchone()
if not match:
match = myDB.action(
'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
[artist['artist_name'], rg['title'], track['title']
]).fetchone()
# if not match:
# match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
if match:
newValueDict['Location'] = match['Location']
newValueDict['BitRate'] = match['BitRate']
newValueDict['Format'] = match['Format']
# myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
myDB.action('UPDATE have SET Matched=? WHERE Location=?',
(rg['id'], match['Location']))
myDB.upsert("alltracks", newValueDict, controlValueDict)
# Delete matched tracks from the have table
# myDB.action('DELETE from have WHERE Matched="True"')
# If there's no release in the main albums tables, add the default (hybrid)
# If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
# check if the album already exists
if not rg_exists:
releaseid = rg['id']
else:
releaseid = rg_exists['ReleaseID']
if not releaseid:
releaseid = rg['id']
album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?',
[releaseid]).fetchone()
controlValueDict = {"AlbumID": rg['id']}
newValueDict = {
"ArtistID": album['ArtistID'],
"ArtistName": album['ArtistName'],
"AlbumTitle": album['AlbumTitle'],
"ReleaseID": album['ReleaseID'],
"AlbumASIN": album['AlbumASIN'],
"ReleaseDate": album['ReleaseDate'],
"Type": album['Type'],
"ReleaseCountry": album['ReleaseCountry'],
"ReleaseFormat": album['ReleaseFormat']
}
if rg_exists:
newValueDict['DateAdded'] = rg_exists['DateAdded']
newValueDict['Status'] = rg_exists['Status']
else:
today = helpers.today()
newValueDict['DateAdded'] = today
if headphones.CONFIG.AUTOWANT_ALL:
newValueDict['Status'] = "Wanted"
elif album[
'ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING:
newValueDict['Status'] = "Wanted"
# Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
# The first test just makes sure we have year-month-day
elif helpers.get_age(
album['ReleaseDate']
) and helpers.get_age(today) - helpers.get_age(
album['ReleaseDate']
) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
newValueDict['Status'] = "Wanted"
else:
newValueDict['Status'] = "Skipped"
myDB.upsert("albums", newValueDict, controlValueDict)
tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?',
[releaseid]).fetchall()
# This is used to see how many tracks you have from an album - to
# mark it as downloaded. Default is 80%, can be set in config as
# ALBUM_COMPLETION_PCT
total_track_count = len(tracks)
if total_track_count == 0:
logger.warning(
"Total track count is zero for Release ID " +
"'%s', skipping.", releaseid)
continue
for track in tracks:
controlValueDict = {
"TrackID": track['TrackID'],
"AlbumID": rg['id']
}
newValueDict = {
"ArtistID": track['ArtistID'],
"ArtistName": track['ArtistName'],
"AlbumTitle": track['AlbumTitle'],
"AlbumASIN": track['AlbumASIN'],
"ReleaseID": track['ReleaseID'],
"TrackTitle": track['TrackTitle'],
"TrackDuration": track['TrackDuration'],
"TrackNumber": track['TrackNumber'],
"CleanName": track['CleanName'],
"Location": track['Location'],
"Format": track['Format'],
"BitRate": track['BitRate']
}
myDB.upsert("tracks", newValueDict, controlValueDict)
# Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
have_track_count = len(
myDB.select(
'SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL',
[rg['id']]))
marked_as_downloaded = False
if rg_exists:
if rg_exists['Status'] == 'Skipped' and (
(have_track_count / float(total_track_count)) >=
(headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?',
['Downloaded', rg['id']])
marked_as_downloaded = True
else:
if (have_track_count / float(total_track_count)) >= (
headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0):
myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?',
['Downloaded', rg['id']])
marked_as_downloaded = True
logger.info(u"[%s] Seeing if we need album art for %s" %
(artist['artist_name'], rg['title']))
cache.getThumb(AlbumID=rg['id'])
# Start a search for the album if it's new, hasn't been marked as
# downloaded and autowant_all is selected. This search is deferred,
# in case the search failes and the rest of the import will halt.
if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL:
album_searches.append(rg['id'])
else:
if skip_log == 0:
logger.info(u"[%s] No new releases, so no changes made to %s" %
(artist['artist_name'], rg['title']))
time.sleep(3)
finalize_update(artistid, artist['artist_name'], errors)
logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
cache.getThumb(ArtistID=artistid)
logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name'])
metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
if errors:
logger.info(
"[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database"
% (artist['artist_name'], artist['artist_name']))
else:
myDB.action('DELETE FROM newartists WHERE ArtistName = ?',
[artist['artist_name']])
logger.info(u"Updating complete for: %s" % artist['artist_name'])
# Start searching for newly added albums
if album_searches:
from headphones import searcher
logger.info("Start searching for %d albums.", len(album_searches))
for album_search in album_searches:
searcher.searchforalbum(albumid=album_search)
