def _is_current(self, filename=None, date=None): if filename: base_filename = os.path.basename(filename) date = base_filename.split('.')[1] # Calculate how old the cached file is based on todays date & file date stamp # helpers.today() returns todays date in yyyy-mm-dd format if self._get_age(helpers.today()) - self._get_age(date) < 30: return True else: return False
def addArtisttoDB(artistid, extrasonly=False): # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return myDB = db.DBConnection() artist = mb.getArtist(artistid, extrasonly) if not artist: return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} if headphones.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist['releasegroups']: rgid = rg['id'] # check if the album already exists rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rg['id']]) try: release_dict = mb.getReleaseGroup(rgid) except Exception, e: logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue if not release_dict: continue logger.info(u"Now adding/updating album: " + rg['title']) controlValueDict = {"AlbumID": rg['id']} if len(rg_exists): newValueDict = {"AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['releasedate'], } else: newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['releasedate'], "DateAdded": helpers.today(), "Type": rg['type'] } if release_dict['releasedate'] > helpers.today(): newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) try: lastfm.getAlbumDescription(rg['id'], artist['artist_name'], rg['title']) except Exception, e: logger.error('Attempt to retrieve album description from Last.fm failed: %s' % e)
artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) if not artist_exists and release_dict: if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: sortname = release_dict['artist_name'] logger.info(u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused") controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = { "ArtistName": release_dict['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused" } if headphones.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 newValueDict['Extras'] = headphones.EXTRAS myDB.upsert("artists", newValueDict, controlValueDict) elif not artist_exists and not release_dict: logger.error( "Artist does not exist in the database and did not get a valid response from MB. Skipping release." ) return
def _update_cache(self): ''' Since we call the same url for both info and artwork, we'll update both at the same time ''' myDB = db.DBConnection() # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums if self.id_type == 'artist': data = lastfm.request_lastfm("artist.getinfo", mbid=self.id, api_key=lastfm_apikey) if not data: return try: self.info_summary = data['artist']['bio']['summary'] except Exception: logger.debug('No artist bio summary found') self.info_summary = None try: self.info_content = data['artist']['bio']['content'] except Exception: logger.debug('No artist bio found') self.info_content = None try: image_url = data['artist']['image'][-1]['#text'] except Exception: logger.debug('No artist image found') image_url = None thumb_url = self._get_thumb_url(data) if not thumb_url: logger.debug('No artist thumbnail image found') else: dbartist = myDB.action('SELECT ArtistName, AlbumTitle FROM albums WHERE AlbumID=?', [self.id]).fetchone() data = lastfm.request_lastfm("album.getinfo", artist=dbartist['ArtistName'], album=dbartist['AlbumTitle'], api_key=lastfm_apikey) if not data: return try: self.info_summary = data['album']['wiki']['summary'] except Exception: logger.debug('No album summary found') self.info_summary = None try: self.info_content = data['album']['wiki']['content'] except Exception: logger.debug('No album infomation found') self.info_content = None try: image_url = data['album']['image'][-1]['#text'] except Exception: logger.debug('No album image link found') image_url = None thumb_url = self._get_thumb_url(data) if not thumb_url: logger.debug('No album thumbnail image found') #Save the content & summary to the database no matter what if we've opened up the url if self.id_type == 'artist': controlValueDict = {"ArtistID": self.id} else: controlValueDict = {"ReleaseGroupID": self.id} newValueDict = {"Summary": self.info_summary, "Content": self.info_content, "LastUpdated": helpers.today()} myDB.upsert("descriptions", newValueDict, controlValueDict) # Save the image URL to the database if image_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ArtworkURL=? WHERE ArtistID=?', [image_url, self.id]) else: myDB.action('UPDATE albums SET ArtworkURL=? WHERE AlbumID=?', [image_url, self.id]) # Save the thumb URL to the database if thumb_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ThumbURL=? WHERE ArtistID=?', [thumb_url, self.id]) else: myDB.action('UPDATE albums SET ThumbURL=? WHERE AlbumID=?', [thumb_url, self.id]) # Should we grab the artwork here if we're just grabbing thumbs or info?? Probably not since the files can be quite big if image_url and self.query_type == 'artwork': artwork = request.request_content(image_url, timeout=20) if artwork: # Make sure the artwork dir exists: if not os.path.isdir(self.path_to_art_cache): try: os.makedirs(self.path_to_art_cache) except Exception, e: logger.error('Unable to create artwork cache dir. Error: %s', e) self.artwork_errors = True self.artwork_url = image_url #Delete the old stuff for artwork_file in self.artwork_files: try: os.remove(artwork_file) except: logger.error('Error deleting file from the cache: %s', artwork_file) ext = os.path.splitext(image_url)[1] artwork_path = os.path.join(self.path_to_art_cache, self.id + '.' + helpers.today() + ext) try: f = open(artwork_path, 'wb') f.write(artwork) f.close() except Exception, e: logger.error('Unable to write to the cache dir: %s', e) self.artwork_errors = True self.artwork_url = image_url
def verify(albumid, albumpath): myDB = db.DBConnection() release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) if not release or not tracks: #the result of a manual post-process on an album that hasn't been inserted #from an RSS feed or etc #TODO: This should be a call to a class method.. copied it out of importer with only minor changes #TODO: odd things can happen when there are diacritic characters in the folder name, need to translate them? import mb release_dict = None try: release_dict = mb.getReleaseGroup(albumid) except Exception, e: logger.info('Unable to get release information for manual album with rgid: %s. Error: %s' % (albumid, e)) return if not release_dict: logger.info('Unable to get release information for manual album with rgid: %s' % albumid) return logger.info(u"Now adding/updating artist: " + release_dict['artist_name']) if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: sortname = release_dict['artist_name'] controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = {"ArtistName": release_dict['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused"} logger.info("ArtistID:ArtistName: " + release_dict['artist_id'] + " : " + release_dict['artist_name']) if headphones.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 myDB.upsert("artists", newValueDict, controlValueDict) logger.info(u"Now adding album: " + release_dict['title']) controlValueDict = {"AlbumID": albumid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['releasedate'], "DateAdded": helpers.today(), "Type": release_dict['type'], "Status": "Snatched" } myDB.upsert("albums", newValueDict, controlValueDict) # I changed the albumid from releaseid -> rgid, so might need to delete albums that have a releaseid for rel in release_dict['releaselist']: myDB.action('DELETE from albums WHERE AlbumID=?', [rel['releaseid']]) myDB.action('DELETE from tracks WHERE AlbumID=?', [rel['releaseid']]) myDB.action('DELETE from tracks WHERE AlbumID=?', [albumid]) for track in release_dict['tracks']: controlValueDict = {"TrackID": track['id'], "AlbumID": albumid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = {"Status": "Paused"} myDB.upsert("artists", newValueDict, controlValueDict) logger.info(u"Addition complete for: " + release_dict['title'] + " - " + release_dict['artist_name']) release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
# make sure the artist exists since I don't know what happens later if it doesn't artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) if not artist_exists and release_dict: if release_dict["artist_name"].startswith("The "): sortname = release_dict["artist_name"][4:] else: sortname = release_dict["artist_name"] logger.info(u"Now manually adding: " + release_dict["artist_name"] + " - with status Paused") controlValueDict = {"ArtistID": release_dict["artist_id"]} newValueDict = { "ArtistName": release_dict["artist_name"], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused", } if headphones.INCLUDE_EXTRAS: newValueDict["IncludeExtras"] = 1 newValueDict["Extras"] = headphones.EXTRAS myDB.upsert("artists", newValueDict, controlValueDict) elif not artist_exists and not release_dict: logger.error( "Artist does not exist in the database and did not get a valid response from MB. Skipping release." ) return
def addArtisttoDB(artistid, extrasonly=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.INCLUDE_EXTRAS, "Extras": headphones.EXTRAS } else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist['releasegroups']: logger.info("Now adding/updating: " + rg['title']) rgid = rg['id'] # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() try: releaselist = mb.getReleaseGroup(rgid) except Exception, e: logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue if not releaselist: errors = True continue # This will be used later to build a hybrid release fullreleaselist = [] for release in releaselist: # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table releaseid = release['id'] try: releasedict = mb.getRelease(releaseid, include_artist_info=False) except Exception, e: errors = True logger.info('Unable to get release information for %s: %s' % (release['id'], e)) continue if not releasedict: errors = True continue controlValueDict = {"ReleaseID": release['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": releasedict['asin'], "ReleaseDate": releasedict['date'], "Type": rg['type'], "ReleaseCountry": releasedict['country'], "ReleaseFormat": releasedict['format'] } myDB.upsert("allalbums", newValueDict, controlValueDict) # Build the dictionary for the fullreleaselist newValueDict['ReleaseID'] = release['id'] newValueDict['Tracks'] = releasedict['tracks'] fullreleaselist.append(newValueDict) for track in releasedict['tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": release['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": releasedict['asin'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict)
def getArtists_json(self,iDisplayStart=0,iDisplayLength=100,sSearch="",iSortCol_0='0',sSortDir_0='asc',**kwargs): iDisplayStart = int(iDisplayStart) iDisplayLength = int(iDisplayLength) filtered = [] totalcount = 0 myDB = db.DBConnection() sortcolumn = 'ArtistSortName' sortbyhavepercent = False if iSortCol_0 == '2': sortcolumn = 'Status' elif iSortCol_0 == '3': sortcolumn = 'ReleaseDate' elif iSortCol_0 == '4': sortbyhavepercent = True if sSearch == "": query = 'SELECT * from artists order by %s COLLATE NOCASE %s' % (sortcolumn,sSortDir_0) filtered = myDB.select(query) totalcount = len(filtered) else: query = 'SELECT * from artists WHERE ArtistSortName LIKE "%' + sSearch + '%" OR LatestAlbum LIKE "%' + sSearch +'%"' + 'ORDER BY %s COLLATE NOCASE %s' % (sortcolumn,sSortDir_0) filtered = myDB.select(query) totalcount = myDB.select('SELECT COUNT(*) from artists')[0][0] if sortbyhavepercent: filtered.sort(key=lambda x:(float(x['HaveTracks'])/x['TotalTracks'] if x['TotalTracks'] > 0 else 0.0,x['HaveTracks'] if x['HaveTracks'] else 0.0),reverse=sSortDir_0 == "asc") #can't figure out how to change the datatables default sorting order when its using an ajax datasource so ill #just reverse it here and the first click on the "Latest Album" header will sort by descending release date if sortcolumn == 'ReleaseDate': filtered.reverse() artists = filtered[iDisplayStart:(iDisplayStart+iDisplayLength)] rows = [] for artist in artists: row = {"ArtistID":artist['ArtistID'], "ArtistSortName":artist["ArtistSortName"], "Status":artist["Status"], "TotalTracks":artist["TotalTracks"], "HaveTracks":artist["HaveTracks"], "LatestAlbum":"", "ReleaseDate":"", "ReleaseInFuture":"False", "AlbumID":"", } if not row['HaveTracks']: row['HaveTracks'] = 0 if artist['ReleaseDate'] and artist['LatestAlbum']: row['ReleaseDate'] = artist['ReleaseDate'] row['LatestAlbum'] = artist['LatestAlbum'] row['AlbumID'] = artist['AlbumID'] if artist['ReleaseDate'] > today(): row['ReleaseInFuture'] = "True" elif artist['LatestAlbum']: row['ReleaseDate'] = '' row['LatestAlbum'] = artist['LatestAlbum'] row['AlbumID'] = artist['AlbumID'] rows.append(row) dict = {'iTotalDisplayRecords':len(filtered), 'iTotalRecords':totalcount, 'aaData':rows, } s = simplejson.dumps(dict) cherrypy.response.headers['Content-type'] = 'application/json' return s
def _update_cache(self): """ Since we call the same url for both info and artwork, we'll update both at the same time """ myDB = db.DBConnection() # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums # Exception is when adding albums manually, then we should use release id if self.id_type == 'artist': data = lastfm.request_lastfm("artist.getinfo", mbid=self.id, api_key=LASTFM_API_KEY) if not data: return try: self.info_summary = data['artist']['bio']['summary'] except KeyError: logger.debug('No artist bio summary found') self.info_summary = None try: self.info_content = data['artist']['bio']['content'] except KeyError: logger.debug('No artist bio found') self.info_content = None try: image_url = data['artist']['image'][-1]['#text'] except KeyError: logger.debug('No artist image found') image_url = None thumb_url = self._get_thumb_url(data) if not thumb_url: logger.debug('No artist thumbnail image found') else: dbalbum = myDB.action( 'SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?', [self.id]).fetchone() if dbalbum['ReleaseID'] != self.id: data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'], api_key=LASTFM_API_KEY) if not data: data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY) else: data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY) if not data: return try: self.info_summary = data['album']['wiki']['summary'] except KeyError: logger.debug('No album summary found') self.info_summary = None try: self.info_content = data['album']['wiki']['content'] except KeyError: logger.debug('No album infomation found') self.info_content = None try: image_url = data['album']['image'][-1]['#text'] except KeyError: logger.debug('No album image link found') image_url = None thumb_url = self._get_thumb_url(data) if not thumb_url: logger.debug('No album thumbnail image found') # Save the content & summary to the database no matter what if we've # opened up the url if self.id_type == 'artist': controlValueDict = {"ArtistID": self.id} else: controlValueDict = {"ReleaseGroupID": self.id} newValueDict = {"Summary": self.info_summary, "Content": self.info_content, "LastUpdated": helpers.today()} myDB.upsert("descriptions", newValueDict, controlValueDict) # Save the image URL to the database if image_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ArtworkURL=? WHERE ArtistID=?', [image_url, self.id]) else: myDB.action('UPDATE albums SET ArtworkURL=? WHERE AlbumID=?', [image_url, self.id]) # Save the thumb URL to the database if thumb_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ThumbURL=? WHERE ArtistID=?', [thumb_url, self.id]) else: myDB.action('UPDATE albums SET ThumbURL=? WHERE AlbumID=?', [thumb_url, self.id]) # Should we grab the artwork here if we're just grabbing thumbs or # info? Probably not since the files can be quite big if image_url and self.query_type == 'artwork': artwork = request.request_content(image_url, timeout=20) if artwork: # Make sure the artwork dir exists: if not os.path.isdir(self.path_to_art_cache): try: os.makedirs(self.path_to_art_cache) os.chmod(self.path_to_art_cache, int(headphones.CONFIG.FOLDER_PERMISSIONS, 8)) except OSError as e: logger.error('Unable to create artwork cache dir. Error: %s', e) self.artwork_errors = True self.artwork_url = image_url # Delete the old stuff for artwork_file in self.artwork_files: try: os.remove(artwork_file) except: logger.error('Error deleting file from the cache: %s', artwork_file) ext = os.path.splitext(image_url)[1] artwork_path = os.path.join(self.path_to_art_cache, self.id + '.' + helpers.today() + ext) try: with open(artwork_path, 'wb') as f: f.write(artwork) os.chmod(artwork_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8)) except (OSError, IOError) as e: logger.error('Unable to write to the cache dir: %s', e) self.artwork_errors = True self.artwork_url = image_url # Grab the thumbnail as well if we're getting the full artwork (as long # as it's missing/outdated. if thumb_url and self.query_type in ['thumb', 'artwork'] and not ( self.thumb_files and self._is_current(self.thumb_files[0])): artwork = request.request_content(thumb_url, timeout=20) if artwork: # Make sure the artwork dir exists: if not os.path.isdir(self.path_to_art_cache): try: os.makedirs(self.path_to_art_cache) os.chmod(self.path_to_art_cache, int(headphones.CONFIG.FOLDER_PERMISSIONS, 8)) except OSError as e: logger.error('Unable to create artwork cache dir. Error: %s' + e) self.thumb_errors = True self.thumb_url = thumb_url # Delete the old stuff for thumb_file in self.thumb_files: try: os.remove(thumb_file) except OSError as e: logger.error('Error deleting file from the cache: %s', thumb_file) ext = os.path.splitext(image_url)[1] thumb_path = os.path.join(self.path_to_art_cache, 'T_' + self.id + '.' + helpers.today() + ext) try: with open(thumb_path, 'wb') as f: f.write(artwork) os.chmod(thumb_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8)) except (OSError, IOError) as e: logger.error('Unable to write to the cache dir: %s', e) self.thumb_errors = True self.thumb_url = image_url
def getArtists_json( self, iDisplayStart=0, iDisplayLength=100, sSearch="", iSortCol_0="0", sSortDir_0="asc", **kwargs ): iDisplayStart = int(iDisplayStart) iDisplayLength = int(iDisplayLength) filtered = [] totalcount = 0 myDB = db.DBConnection() sortcolumn = "ArtistSortName" sortbyhavepercent = False if iSortCol_0 == "2": sortcolumn = "Status" elif iSortCol_0 == "3": sortcolumn = "ReleaseDate" elif iSortCol_0 == "4": sortbyhavepercent = True if sSearch == "": query = "SELECT * from artists order by %s COLLATE NOCASE %s" % (sortcolumn, sSortDir_0) filtered = myDB.select(query) totalcount = len(filtered) else: query = ( 'SELECT * from artists WHERE ArtistSortName LIKE "%' + sSearch + '%" OR LatestAlbum LIKE "%' + sSearch + '%"' + "ORDER BY %s COLLATE NOCASE %s" % (sortcolumn, sSortDir_0) ) filtered = myDB.select(query) totalcount = myDB.select("SELECT COUNT(*) from artists")[0][0] if sortbyhavepercent: filtered.sort( key=lambda x: ( float(x["HaveTracks"]) / x["TotalTracks"] if x["TotalTracks"] > 0 else 0.0, x["HaveTracks"] if x["HaveTracks"] else 0.0, ), reverse=sSortDir_0 == "asc", ) # can't figure out how to change the datatables default sorting order when its using an ajax datasource so ill # just reverse it here and the first click on the "Latest Album" header will sort by descending release date if sortcolumn == "ReleaseDate": filtered.reverse() artists = filtered[iDisplayStart : (iDisplayStart + iDisplayLength)] rows = [] for artist in artists: row = { "ArtistID": artist["ArtistID"], "ArtistSortName": artist["ArtistSortName"], "Status": artist["Status"], "TotalTracks": artist["TotalTracks"], "HaveTracks": artist["HaveTracks"], "LatestAlbum": "", "ReleaseDate": "", "ReleaseInFuture": "False", "AlbumID": "", } if not row["HaveTracks"]: row["HaveTracks"] = 0 if artist["ReleaseDate"] and artist["LatestAlbum"]: row["ReleaseDate"] = artist["ReleaseDate"] row["LatestAlbum"] = artist["LatestAlbum"] row["AlbumID"] = artist["AlbumID"] if artist["ReleaseDate"] > today(): row["ReleaseInFuture"] = "True" elif artist["LatestAlbum"]: row["ReleaseDate"] = "" row["LatestAlbum"] = artist["LatestAlbum"] row["AlbumID"] = artist["AlbumID"] rows.append(row) dict = {"iTotalDisplayRecords": len(filtered), "iTotalRecords": totalcount, "aaData": rows} s = simplejson.dumps(dict) cherrypy.response.headers["Content-type"] = "application/json" return s
def addArtisttoDB(artistid, extrasonly=False): # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn("Cannot import Various Artists.") return myDB = db.DBConnection() artist = mb.getArtist(artistid, extrasonly) if not artist: return if artist["artist_name"].startswith("The "): sortname = artist["artist_name"][4:] else: sortname = artist["artist_name"] logger.info(u"Now adding/updating: " + artist["artist_name"]) controlValueDict = {"ArtistID": artistid} newValueDict = { "ArtistName": artist["artist_name"], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading", } if headphones.INCLUDE_EXTRAS: newValueDict["IncludeExtras"] = 1 myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist["releasegroups"]: rgid = rg["id"] # check if the album already exists rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rg["id"]]) try: release_dict = mb.getReleaseGroup(rgid) except Exception, e: logger.info( "Unable to get release information for %s - there may not be any official releases in this release group" % rg["title"] ) continue if not release_dict: continue logger.info(u"Now adding/updating album: " + rg["title"]) controlValueDict = {"AlbumID": rg["id"]} if len(rg_exists): newValueDict = {"AlbumASIN": release_dict["asin"], "ReleaseDate": release_dict["releasedate"]} else: newValueDict = { "ArtistID": artistid, "ArtistName": artist["artist_name"], "AlbumTitle": rg["title"], "AlbumASIN": release_dict["asin"], "ReleaseDate": release_dict["releasedate"], "DateAdded": helpers.today(), "Type": rg["type"], } if release_dict["releasedate"] > helpers.today(): newValueDict["Status"] = "Wanted" else: newValueDict["Status"] = "Skipped" if rg["type"] == "Album" and headphones.AUTOWANT_ALBUM: newValueDict["Status"] = "Wanted" if rg["type"] == "Compilation" and headphones.AUTOWANT_COMPILATION: newValueDict["Status"] = "Wanted" if rg["type"] == "EP" and headphones.AUTOWANT_EP: newValueDict["Status"] = "Wanted" if rg["type"] == "Remix" and headphones.AUTOWANT_REMIX: newValueDict["Status"] = "Wanted" if rg["type"] == "Single" and headphones.AUTOWANT_SINGLE: newValueDict["Status"] = "Wanted" if rg["type"] == "Live" and headphones.AUTOWANT_LIVE: newValueDict["Status"] = "Wanted" if rg["type"] == "Soundtrack" and headphones.AUTOWANT_SOUNDTRACK: newValueDict["Status"] = "Wanted" myDB.upsert("albums", newValueDict, controlValueDict) try: lastfm.getAlbumDescription(rg["id"], artist["artist_name"], rg["title"]) except Exception, e: logger.error("Attempt to retrieve album description from Last.fm failed: %s" % e)
os.makedirs(self.path_to_art_cache) except Exception, e: logger.error("Unable to create artwork cache dir. Error: " + str(e)) self.thumb_errors = True self.thumb_url = thumb_url # Delete the old stuff for thumb_file in self.thumb_files: try: os.remove(thumb_file) except: logger.error("Error deleting file from the cache: " + thumb_file) ext = os.path.splitext(image_url)[1] thumb_path = os.path.join(self.path_to_art_cache, "T_" + self.id + "." + helpers.today() + ext) try: f = open(thumb_path, "wb") f.write(artwork) f.close() except Exception, e: logger.error("Unable to write to the cache dir: " + str(e)) self.thumb_errors = True self.thumb_url = image_url def getArtwork(ArtistID=None, AlbumID=None): c = Cache() artwork_path = c.get_artwork_from_cache(ArtistID, AlbumID)
def addArtisttoDB(artistid, extrasonly=False): # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return myDB = db.DBConnection() artist = mb.getArtist(artistid, extrasonly) if not artist: return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} if headphones.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist['releasegroups']: rgid = rg['id'] # check if the album already exists rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rg['id']]) try: release_dict = mb.getReleaseGroup(rgid) except Exception, e: logger.info('Unable to get release information for %s - it may not be a valid release group (or it might just not be tagged right in MusicBrainz)' % rg['title']) continue if not release_dict: continue logger.info(u"Now adding/updating album: " + rg['title']) controlValueDict = {"AlbumID": rg['id']} if len(rg_exists): newValueDict = {"AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['releasedate'], } else: newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['releasedate'], "DateAdded": helpers.today(), "Type": rg['type'] } if release_dict['releasedate'] > helpers.today(): newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) # I changed the albumid from releaseid -> rgid, so might need to delete albums that have a releaseid for release in release_dict['releaselist']: myDB.action('DELETE from albums WHERE AlbumID=?', [release['releaseid']]) myDB.action('DELETE from tracks WHERE AlbumID=?', [release['releaseid']]) myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) for track in release_dict['tracks']: controlValueDict = {"TrackID": track['id'], "AlbumID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'] } myDB.upsert("tracks", newValueDict, controlValueDict)
def addArtisttoDB(artistid, extrasonly=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = { "ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.INCLUDE_EXTRAS, "Extras": headphones.EXTRAS } else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = { "ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active" } else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = { "ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading" } myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence over global option # Global options are set when adding a new artist myDB = db.DBConnection() try: db_artist = myDB.action( 'SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False for rg in artist['releasegroups']: logger.info("Now adding/updating: " + rg['title']) rgid = rg['id'] # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() releases = mb.get_all_releases(rgid, includeExtras) if releases == []: logger.info('No official releases in release group %s' % rg['title']) continue if not releases: errors = True logger.info( 'Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue # This will be used later to build a hybrid release fullreleaselist = [] for release in releases: # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table controlValueDict = {"ReleaseID": release['ReleaseID']} newValueDict = { "ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "ReleaseDate": release['ReleaseDate'], "Type": release['Type'], "ReleaseCountry": release['ReleaseCountry'], "ReleaseFormat": release['ReleaseFormat'] } myDB.upsert("allalbums", newValueDict, controlValueDict) # Build the dictionary for the fullreleaselist newValueDict['ReleaseID'] = release['ReleaseID'] newValueDict['Tracks'] = release['Tracks'] fullreleaselist.append(newValueDict) for track in release['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = { "TrackID": track['id'], "ReleaseID": release['ReleaseID'] } newValueDict = { "ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title'] ]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action( 'UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) except Exception, e: errors = True logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'], e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg['id']} newValueDict = { "ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": rg['id']} newValueDict = { "ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title'] ]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) if not rg_exists: releaseid = rg['id'] elif rg_exists and not rg_exists['ReleaseID']: # Need to do some importing here - to transition the old format of using the release group # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID # so we'll need to move over the locations, bitrates & formats from the tracks table to the new # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups logger.info( "Copying current track information to alternate releases") tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() for track in tracks: if track['Location']: controlValueDict = {"TrackID": track['TrackID']} newValueDict = { "Location": track['Location'], "BitRate": track['BitRate'], "Format": track['Format'], } myDB.upsert("alltracks", newValueDict, controlValueDict) releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} newValueDict = { "ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if not rg_exists: newValueDict['DateAdded'] = helpers.today() if headphones.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif album['ReleaseDate'] > helpers.today( ) and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) #start a search for the album if it's new and autowant_all is selected: # Should this run in a background thread? Don't know if we want to have a bunch of # simultaneous threads running if not rg_exists and headphones.AUTOWANT_ALL: from headphones import searcher searcher.searchforalbum(albumid=rg['id']) myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT total_track_count = len(tracks) for track in tracks: controlValueDict = { "TrackID": track['TrackID'], "AlbumID": rg['id'] } newValueDict = { "ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len( myDB.select( 'SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) if rg_exists: if rg_exists['Status'] == 'Skipped' and ( (have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) else: if ((have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) logger.info(u"Seeing if we need album art for " + rg['title']) cache.getThumb(AlbumID=rg['id'])
def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn('Cannot import blocked special purpose artist with id' + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS, "Extras": headphones.CONFIG.EXTRAS} else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if artist and artist.get('artist_name') in blacklisted_special_artist_names: logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name')) myDB.action('DELETE from artists WHERE ArtistID=?', [artistid]) #in case it's already in the db myDB.action('DELETE from albums WHERE ArtistID=?', [artistid]) myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence # over global option. Global options are set when adding a new artist try: db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False # Clean all references to release group in dB that are no longer referenced # from the musicbrainz refresh group_list = [] force_repackage = 0 # Don't nuke the database if there's a MusicBrainz error if len(artist['releasegroups']) != 0: for groups in artist['releasegroups']: group_list.append(groups['id']) if not extrasonly: remove_missing_groups_from_albums = myDB.select("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) else: remove_missing_groups_from_albums = myDB.select('SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid]) for items in remove_missing_groups_from_albums: if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']]) logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (artist['artist_name'], items['AlbumID'])) if not extrasonly: force_repackage = 1 else: if not extrasonly: logger.info("[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name']) # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks album_searches = [] for rg in artist['releasegroups']: al_title = rg['title'] today = helpers.today() rgid = rg['id'] skip_log = 0 #Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.CONFIG.MB_IGNORE_AGE rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() if not forcefull: new_release_group = False try: check_release_date = rg_exists['ReleaseDate'] except TypeError: check_release_date = None new_release_group = True if new_release_group: logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid, includeExtras) else: if check_release_date is None or check_release_date == u"None": logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: if len(check_release_date) == 10: release_date = check_release_date elif len(check_release_date) == 7: release_date = check_release_date + "-31" elif len(check_release_date) == 4: release_date = check_release_date + "-12-31" else: release_date = today if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta: logger.info("[%s] Now updating: %s (Release Date <%s Days)", artist['artist_name'], rg['title'], pause_delta) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: logger.info("[%s] Skipping: %s (Release Date >%s Days)", artist['artist_name'], rg['title'], pause_delta) skip_log = 1 new_releases = 0 if force_repackage == 1: new_releases = -1 logger.info('[%s] Forcing repackage of %s (Release Group Removed)', artist['artist_name'], al_title) else: new_releases = new_releases else: logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'], rg['title']) new_releases = mb.get_new_releases(rgid, includeExtras, forcefull) if new_releases != 0: # Dump existing hybrid release since we're repackaging/replacing it myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rg['id']]) # This will be used later to build a hybrid release fullreleaselist = [] # Search for releases within a release group find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) # Build the dictionary for the fullreleaselist for items in find_hybrid_releases: if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing hybrid_release_id = items['ReleaseID'] newValueDict = {"ArtistID": items['ArtistID'], "ArtistName": items['ArtistName'], "AlbumTitle": items['AlbumTitle'], "AlbumID": items['AlbumID'], "AlbumASIN": items['AlbumASIN'], "ReleaseDate": items['ReleaseDate'], "Type": items['Type'], "ReleaseCountry": items['ReleaseCountry'], "ReleaseFormat": items['ReleaseFormat'] } find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] for hybrid_tracks in find_hybrid_tracks: hybrid_track_array.append({ 'number': hybrid_tracks['TrackNumber'], 'title': hybrid_tracks['TrackTitle'], 'id': hybrid_tracks['TrackID'], #'url': hybrid_tracks['TrackURL'], 'duration': hybrid_tracks['TrackDuration'] }) totalTracks += 1 newValueDict['ReleaseID'] = hybrid_release_id newValueDict['Tracks'] = hybrid_track_array fullreleaselist.append(newValueDict) # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title'])) except Exception as e: errors = True logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'], rg['title'], e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() #if not match: #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table #myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) # check if the album already exists if not rg_exists: releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] if not releaseid: releaseid = rg['id'] album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if rg_exists: newValueDict['DateAdded'] = rg_exists['DateAdded'] newValueDict['Status'] = rg_exists['Status'] else: today = helpers.today() newValueDict['DateAdded'] = today if headphones.CONFIG.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to # mark it as downloaded. Default is 80%, can be set in config as # ALBUM_COMPLETION_PCT total_track_count = len(tracks) if total_track_count == 0: logger.warning("Total track count is zero for Release ID " + "'%s', skipping.", releaseid) continue for track in tracks: controlValueDict = {"TrackID": track['TrackID'], "AlbumID": rg['id']} newValueDict = {"ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) marked_as_downloaded = False if rg_exists: if rg_exists['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True else: if ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title'])) cache.getThumb(AlbumID=rg['id']) # Start a search for the album if it's new, hasn't been marked as # downloaded and autowant_all is selected. This search is deferred, # in case the search failes and the rest of the import will halt. if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL: album_searches.append(rg['id']) else: if skip_log == 0: logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title'])) time.sleep(3) finalize_update(artistid, artist['artist_name'], errors) logger.info(u"Seeing if we need album art for: %s" % artist['artist_name']) cache.getThumb(ArtistID=artistid) if errors: logger.info("[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name'])) else: myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']]) logger.info(u"Updating complete for: %s" % artist['artist_name']) # Start searching for newly added albums if album_searches: from headphones import searcher logger.info("Start searching for %d albums.", len(album_searches)) for album_search in album_searches: searcher.searchforalbum(albumid=album_search)
def addReleaseById(rid, rgid=None): myDB = db.DBConnection() # Create minimum info upfront if added from searchresults status = '' if rgid: dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) if not dbalbum: status = 'Loading' controlValueDict = {"AlbumID": rgid} newValueDict = {"AlbumTitle": rgid, "ArtistName": status, "Status": status} myDB.upsert("albums", newValueDict, controlValueDict) time.sleep(1) rgid = None artistid = None release_dict = None results = myDB.select("SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid]) for result in results: rgid = result['ReleaseGroupID'] artistid = result['ArtistID'] logger.debug("Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid) if not rgid: #didn't find it in the cache, get the information from MB logger.debug("Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID") try: release_dict = mb.getRelease(rid) except Exception as e: logger.info('Unable to get release information for Release %s: %s', rid, e) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not release_dict: logger.info('Unable to get release information for Release %s: no dict', rid) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return rgid = release_dict['rgid'] artistid = release_dict['artist_id'] #we don't want to make more calls to MB here unless we have to, could be happening quite a lot rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) #make sure the artist exists since I don't know what happens later if it doesn't artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) if not artist_exists and release_dict: if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: sortname = release_dict['artist_name'] logger.info(u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused") controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = {"ArtistName": release_dict['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused"} if headphones.CONFIG.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 newValueDict['Extras'] = headphones.CONFIG.EXTRAS myDB.upsert("artists", newValueDict, controlValueDict) elif not artist_exists and not release_dict: logger.error("Artist does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not rg_exists and release_dict or status == 'Loading' and release_dict: #it should never be the case that we have an rg and not the artist #but if it is this will fail logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid) controlValueDict = {"AlbumID": rgid} if status != 'Loading': status = 'Wanted' newValueDict = {"ArtistID": release_dict['artist_id'], "ReleaseID": rgid, "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'] if 'title' in release_dict else release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['date'], "DateAdded": helpers.today(), "Status": status, "Type": release_dict['rg_type'], "ReleaseID": rid } myDB.upsert("albums", newValueDict, controlValueDict) #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "AlbumID": rgid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() #if not match: #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] #myDB.action('DELETE from have WHERE Location=?', [match['Location']]) # If the album has been scanned before adding the release it will be unmatched, update to matched if match['Matched'] == 'Failed': myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release_dict['rgid'], match['Location'])) myDB.upsert("tracks", newValueDict, controlValueDict) # Reset status if status == 'Loading': controlValueDict = {"AlbumID": rgid} if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: newValueDict = {"Status": "Wanted"} else: newValueDict = {"Status": "Skipped"} myDB.upsert("albums", newValueDict, controlValueDict) # Start a search for the album if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: import searcher searcher.searchforalbum(rgid, False) elif not rg_exists and not release_dict: logger.error("ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return else: logger.info('Release ' + str(rid) + " already exists in the database!")
def _update_cache(self): ''' Since we call the same url for both info and artwork, we'll update both at the same time ''' myDB = db.DBConnection() # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums if self.id_type == 'artist': params = { "method": "artist.getInfo", "api_key": lastfm_apikey, "mbid": self.id, "format": "json" } url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params) logger.debug('Retrieving artist information from: ' + url) try: result = urllib2.urlopen(url, timeout=20).read() except: logger.warn('Could not open url: ' + url) return if result: try: data = simplejson.JSONDecoder().decode(result) except: logger.warn('Could not parse data from url: ' + url) return try: self.info_summary = data['artist']['bio']['summary'] except KeyError: logger.debug('No artist bio summary found on url: ' + url) self.info_summary = None try: self.info_content = data['artist']['bio']['content'] except KeyError: logger.debug('No artist bio found on url: ' + url) self.info_content = None try: image_url = data['artist']['image'][-1]['#text'] except KeyError: logger.debug('No artist image found on url: ' + url) image_url = None thumb_url = self._get_thumb_url(data) if not thumb_url: logger.debug('No artist thumbnail image found on url: ' + url) else: dbartist = myDB.action('SELECT ArtistName, AlbumTitle FROM albums WHERE AlbumID=?', [self.id]).fetchone() params = { "method": "album.getInfo", "api_key": lastfm_apikey, "artist": dbartist['ArtistName'].encode('utf-8'), "album": dbartist['AlbumTitle'].encode('utf-8'), "format": "json" } url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params) logger.debug('Retrieving artist information from: ' + url) try: result = urllib2.urlopen(url, timeout=20).read() except: logger.warn('Could not open url: ' + url) return if result: try: data = simplejson.JSONDecoder().decode(result) except: logger.warn('Could not parse data from url: ' + url) return try: self.info_summary = data['album']['wiki']['summary'] except KeyError: logger.debug('No album summary found from: ' + url) self.info_summary = None try: self.info_content = data['album']['wiki']['content'] except KeyError: logger.debug('No album infomation found from: ' + url) self.info_content = None try: image_url = data['album']['image'][-1]['#text'] except KeyError: logger.debug('No album image link found on url: ' + url) image_url = None thumb_url = self._get_thumb_url(data) if not thumb_url: logger.debug('No album thumbnail image found on url: ' + url) #Save the content & summary to the database no matter what if we've opened up the url if self.id_type == 'artist': controlValueDict = {"ArtistID": self.id} else: controlValueDict = {"ReleaseGroupID": self.id} newValueDict = {"Summary": self.info_summary, "Content": self.info_content, "LastUpdated": helpers.today()} myDB.upsert("descriptions", newValueDict, controlValueDict) # Save the image URL to the database if image_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ArtworkURL=? WHERE ArtistID=?', [image_url, self.id]) else: myDB.action('UPDATE albums SET ArtworkURL=? WHERE AlbumID=?', [image_url, self.id]) # Save the thumb URL to the database if thumb_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ThumbURL=? WHERE ArtistID=?', [thumb_url, self.id]) else: myDB.action('UPDATE albums SET ThumbURL=? WHERE AlbumID=?', [thumb_url, self.id]) # Should we grab the artwork here if we're just grabbing thumbs or info?? Probably not since the files can be quite big if image_url and self.query_type == 'artwork': try: artwork = urllib2.urlopen(image_url, timeout=20).read() except Exception, e: logger.error('Unable to open url "' + image_url + '". Error: ' + str(e)) artwork = None if artwork: # Make sure the artwork dir exists: if not os.path.isdir(self.path_to_art_cache): try: os.makedirs(self.path_to_art_cache) except Exception, e: logger.error('Unable to create artwork cache dir. Error: ' + str(e)) self.artwork_errors = True self.artwork_url = image_url #Delete the old stuff for artwork_file in self.artwork_files: try: os.remove(artwork_file) except: logger.error('Error deleting file from the cache: ' + artwork_file) ext = os.path.splitext(image_url)[1] artwork_path = os.path.join(self.path_to_art_cache, self.id + '.' + helpers.today() + ext) try: f = open(artwork_path, 'wb') f.write(artwork) f.close() except Exception, e: logger.error('Unable to write to the cache dir: ' + str(e)) self.artwork_errors = True self.artwork_url = image_url
controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if not rg_exists: newValueDict['DateAdded']= helpers.today() if headphones.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" #start a search for the album import searcher searcher.searchforalbum(albumid=rg['id']) elif album['ReleaseDate'] > helpers.today() and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict)
def addArtisttoDB(artistid, extrasonly=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() if dbartist is None: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading"} else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} if headphones.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist['releasegroups']: rgid = rg['id'] # check if the album already exists rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rg['id']]) try: release_dict = mb.getReleaseGroup(rgid) except Exception, e: logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue if not release_dict: continue logger.info(u"Now adding/updating album: " + rg['title']) controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['releasedate'], "Type": rg['type'] } # Only change the status & add DateAdded if the album is not already in the database if not len(rg_exists): newValueDict['DateAdded']= helpers.today() if headphones.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif release_dict['releasedate'] > helpers.today() and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) for track in release_dict['tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "AlbumID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) logger.debug(u"Updating album cache for " + rg['title']) cache.getThumb(AlbumID=rg['id'])
def verify(albumid, albumpath, Kind=None, forced=False): myDB = db.DBConnection() release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) if not release or not tracks: #the result of a manual post-process on an album that hasn't been inserted #from an RSS feed or etc #TODO: This should be a call to a class method.. copied it out of importer with only minor changes #TODO: odd things can happen when there are diacritic characters in the folder name, need to translate them? import mb release_list = None try: release_list = mb.getReleaseGroup(albumid) except Exception, e: logger.info('Unable to get release information for manual album with rgid: %s. Error: %s' % (albumid, e)) return if not release_list: logger.info('Unable to get release information for manual album with rgid: %s' % albumid) return # Since we're just using this to create the bare minimum information to insert an artist/album combo, use the first release releaseid = release_list[0]['id'] release_dict = mb.getRelease(releaseid) logger.info(u"Now adding/updating artist: " + release_dict['artist_name']) if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: sortname = release_dict['artist_name'] controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = {"ArtistName": release_dict['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused"} logger.info("ArtistID: " + release_dict['artist_id'] + " , ArtistName: " + release_dict['artist_name']) if headphones.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 newValueDict['Extras'] = headphones.EXTRAS myDB.upsert("artists", newValueDict, controlValueDict) logger.info(u"Now adding album: " + release_dict['title']) controlValueDict = {"AlbumID": albumid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['date'], "DateAdded": helpers.today(), "Type": release_dict['rg_type'], "Status": "Snatched" } myDB.upsert("albums", newValueDict, controlValueDict) # Delete existing tracks associated with this AlbumID since we're going to replace them and don't want any extras myDB.action('DELETE from tracks WHERE AlbumID=?', [albumid]) for track in release_dict['tracks']: controlValueDict = {"TrackID": track['id'], "AlbumID": albumid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = {"Status": "Paused"} myDB.upsert("artists", newValueDict, controlValueDict) logger.info(u"Addition complete for: " + release_dict['title'] + " - " + release_dict['artist_name']) release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
def addArtisttoDB(artistid): if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return myDB = db.DBConnection() artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', [artistid]) if any(artistid in x for x in artistlist): logger.info(artistlist[0][1] + u" is already in the database, skipping") return artist = mb.getArtist(artistid) if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist['releasegroups']: rgid = rg['id'] try: releaseid = mb.getReleaseGroup(rgid) except Exception, e: logger.info('Unable to get release information for %s - it may not be a valid release group' % rg['title']) continue release = mb.getRelease(releaseid) logger.info(u"Now adding album: " + release['title']+ " to the database") controlValueDict = {"AlbumID": release['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": release['asin'], "ReleaseDate": release['date'], "DateAdded": helpers.today(), "Status": "Skipped" } myDB.upsert("albums", newValueDict, controlValueDict) latestrelease = myDB.select("SELECT ReleaseDate, DateAdded from albums WHERE AlbumID=?", [release['id']]) if latestrelease[0][0] > latestrelease[0][1]: logger.info(release['title'] + u" is an upcoming album. Setting its status to 'Wanted'...") controlValueDict = {"AlbumID": release['id']} newValueDict = {"Status": "Wanted"} myDB.upsert("albums", newValueDict, controlValueDict) for track in release['tracks']: myDB.action('INSERT INTO tracks VALUES( ?, ?, ?, ?, ?, ?, ?, ?)', [artistid, artist['artist_name'], rg['title'], release['asin'], release['id'], track['title'], track['duration'], track['id']])
def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn("Cannot import blocked special purpose artist with id" + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action("DELETE from blacklist WHERE ArtistID=?", [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action("SELECT * FROM artists WHERE ArtistID=?", [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = { "ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.INCLUDE_EXTRAS, "Extras": headphones.EXTRAS, } else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if artist and artist.get("artist_name") in blacklisted_special_artist_names: logger.warn("Cannot import blocked special purpose artist: %s" % artist.get("artist_name")) myDB.action("DELETE from artists WHERE ArtistID=?", [artistid]) # in case it's already in the db myDB.action("DELETE from albums WHERE ArtistID=?", [artistid]) myDB.action("DELETE from tracks WHERE ArtistID=?", [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist["artist_name"].startswith("The "): sortname = artist["artist_name"][4:] else: sortname = artist["artist_name"] logger.info(u"Now adding/updating: " + artist["artist_name"]) controlValueDict = {"ArtistID": artistid} newValueDict = { "ArtistName": artist["artist_name"], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading", } myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence over global option # Global options are set when adding a new artist myDB = db.DBConnection() try: db_artist = myDB.action("SELECT IncludeExtras, Extras from artists WHERE ArtistID=?", [artistid]).fetchone() includeExtras = db_artist["IncludeExtras"] except IndexError: includeExtras = False # Clean all references to release group in dB that are no longer referenced in musicbrainz group_list = [] force_repackage = 0 # Don't nuke the database if there's a MusicBrainz error if len(artist["releasegroups"]) != 0 and not extrasonly: for groups in artist["releasegroups"]: group_list.append(groups["id"]) remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) for items in remove_missing_groups_from_albums: if items["AlbumID"] not in group_list: # Remove all from albums/tracks that aren't in release groups myDB.action("DELETE FROM albums WHERE AlbumID=?", [items["AlbumID"]]) myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items["AlbumID"]]) myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items["AlbumID"]]) myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items["AlbumID"]]) logger.info( "[%s] Removing all references to release group %s to reflect MusicBrainz" % (artist["artist_name"], items["AlbumID"]) ) force_repackage = 1 else: logger.info("[%s] Error pulling data from MusicBrainz: Maintaining dB" % artist["artist_name"]) # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks for rg in artist["releasegroups"]: al_title = rg["title"] today = helpers.today() rgid = rg["id"] skip_log = 0 # Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.MB_IGNORE_AGE check_release_date = myDB.action( "SELECT ReleaseDate, Status from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title) ).fetchone() # Skip update if Status set if check_release_date and check_release_date[1]: logger.info( "[%s] Not updating: %s (Status is %s, skipping)" % (artist["artist_name"], rg["title"], check_release_date[1]) ) continue if not forcefull: if check_release_date: if check_release_date[0] is None: logger.info("[%s] Now updating: %s (No Release Date)" % (artist["artist_name"], rg["title"])) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: if len(check_release_date[0]) == 10: release_date = check_release_date[0] elif len(check_release_date[0]) == 7: release_date = check_release_date[0] + "-31" elif len(check_release_date[0]) == 4: release_date = check_release_date[0] + "-12-31" else: release_date = today if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta: logger.info( "[%s] Now updating: %s (Release Date <%s Days) " % (artist["artist_name"], rg["title"], pause_delta) ) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: logger.info( "[%s] Skipping: %s (Release Date >%s Days)" % (artist["artist_name"], rg["title"], pause_delta) ) skip_log = 1 new_releases = 0 else: logger.info("[%s] Now adding: %s (New Release Group)" % (artist["artist_name"], rg["title"])) new_releases = mb.get_new_releases(rgid, includeExtras) if force_repackage == 1: new_releases = -1 logger.info("[%s] Forcing repackage of %s (Release Group Removed)" % (artist["artist_name"], al_title)) else: new_releases = new_releases else: logger.info("[%s] Now adding/updating: %s (Comprehensive Force)" % (artist["artist_name"], rg["title"])) new_releases = mb.get_new_releases(rgid, includeExtras, forcefull) # What this does is adds new releases per artist to the allalbums + alltracks databases # new_releases = mb.get_new_releases(rgid,includeExtras) # print al_title # print new_releases if new_releases != 0: # Dump existing hybrid release since we're repackaging/replacing it myDB.action("DELETE from albums WHERE ReleaseID=?", [rg["id"]]) myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg["id"]]) myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg["id"]]) myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg["id"]]) # This will be used later to build a hybrid release fullreleaselist = [] # Search for releases within a release group find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg["id"]]) # Build the dictionary for the fullreleaselist for items in find_hybrid_releases: if ( items["ReleaseID"] != rg["id"] ): # don't include hybrid information, since that's what we're replacing hybrid_release_id = items["ReleaseID"] newValueDict = { "ArtistID": items["ArtistID"], "ArtistName": items["ArtistName"], "AlbumTitle": items["AlbumTitle"], "AlbumID": items["AlbumID"], "AlbumASIN": items["AlbumASIN"], "ReleaseDate": items["ReleaseDate"], "Type": items["Type"], "ReleaseCountry": items["ReleaseCountry"], "ReleaseFormat": items["ReleaseFormat"], } find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] for hybrid_tracks in find_hybrid_tracks: hybrid_track_array.append( { "number": hybrid_tracks["TrackNumber"], "title": hybrid_tracks["TrackTitle"], "id": hybrid_tracks["TrackID"], #'url': hybrid_tracks['TrackURL'], "duration": hybrid_tracks["TrackDuration"], } ) totalTracks += 1 newValueDict["ReleaseID"] = hybrid_release_id newValueDict["Tracks"] = hybrid_track_array fullreleaselist.append(newValueDict) # print fullreleaselist # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) logger.info("[%s] Packaging %s releases into hybrid title" % (artist["artist_name"], rg["title"])) except Exception, e: errors = True logger.warn( "[%s] Unable to get hybrid release information for %s: %s" % (artist["artist_name"], rg["title"], e) ) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg["id"]} newValueDict = { "ArtistID": artistid, "ArtistName": artist["artist_name"], "AlbumTitle": rg["title"], "AlbumID": rg["id"], "AlbumASIN": hybridrelease["AlbumASIN"], "ReleaseDate": hybridrelease["ReleaseDate"], "Type": rg["type"], } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease["Tracks"]: cleanname = helpers.cleanName(artist["artist_name"] + " " + rg["title"] + " " + track["title"]) controlValueDict = {"TrackID": track["id"], "ReleaseID": rg["id"]} newValueDict = { "ArtistID": artistid, "ArtistName": artist["artist_name"], "AlbumTitle": rg["title"], "AlbumASIN": hybridrelease["AlbumASIN"], "AlbumID": rg["id"], "TrackTitle": track["title"], "TrackDuration": track["duration"], "TrackNumber": track["number"], "CleanName": cleanname, } match = myDB.action( "SELECT Location, BitRate, Format from have WHERE CleanName=?", [cleanname] ).fetchone() if not match: match = myDB.action( "SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?", [artist["artist_name"], rg["title"], track["title"]], ).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict["Location"] = match["Location"] newValueDict["BitRate"] = match["BitRate"] newValueDict["Format"] = match["Format"] # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action("UPDATE have SET Matched=? WHERE Location=?", (rg["id"], match["Location"])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table # myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg["id"]]).fetchone() if not rg_exists: releaseid = rg["id"] else: releaseid = rg_exists["ReleaseID"] album = myDB.action("SELECT * from allalbums WHERE ReleaseID=?", [releaseid]).fetchone() controlValueDict = {"AlbumID": rg["id"]} newValueDict = { "ArtistID": album["ArtistID"], "ArtistName": album["ArtistName"], "AlbumTitle": album["AlbumTitle"], "ReleaseID": album["ReleaseID"], "AlbumASIN": album["AlbumASIN"], "ReleaseDate": album["ReleaseDate"], "Type": album["Type"], "ReleaseCountry": album["ReleaseCountry"], "ReleaseFormat": album["ReleaseFormat"], } if not rg_exists: today = helpers.today() newValueDict["DateAdded"] = today if headphones.AUTOWANT_ALL: newValueDict["Status"] = "Wanted" elif album["ReleaseDate"] > today and headphones.AUTOWANT_UPCOMING: newValueDict["Status"] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif ( helpers.get_age(album["ReleaseDate"]) and helpers.get_age(today) - helpers.get_age(album["ReleaseDate"]) < 21 and headphones.AUTOWANT_UPCOMING ): newValueDict["Status"] = "Wanted" else: newValueDict["Status"] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT total_track_count = len(tracks) for track in tracks: controlValueDict = {"TrackID": track["TrackID"], "AlbumID": rg["id"]} newValueDict = { "ArtistID": track["ArtistID"], "ArtistName": track["ArtistName"], "AlbumTitle": track["AlbumTitle"], "AlbumASIN": track["AlbumASIN"], "ReleaseID": track["ReleaseID"], "TrackTitle": track["TrackTitle"], "TrackDuration": track["TrackDuration"], "TrackNumber": track["TrackNumber"], "CleanName": track["CleanName"], "Location": track["Location"], "Format": track["Format"], "BitRate": track["BitRate"], } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len( myDB.select("SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL", [rg["id"]]) ) marked_as_downloaded = False if rg_exists: if rg_exists["Status"] == "Skipped" and ( (have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0) ): myDB.action("UPDATE albums SET Status=? WHERE AlbumID=?", ["Downloaded", rg["id"]]) marked_as_downloaded = True else: if (have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0): myDB.action("UPDATE albums SET Status=? WHERE AlbumID=?", ["Downloaded", rg["id"]]) marked_as_downloaded = True logger.info(u"[%s] Seeing if we need album art for %s" % (artist["artist_name"], rg["title"])) cache.getThumb(AlbumID=rg["id"]) # start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: from headphones import searcher searcher.searchforalbum(albumid=rg["id"]) else: if skip_log == 0: logger.info(u"[%s] No new releases, so no changes made to %s" % (artist["artist_name"], rg["title"]))
def addArtisttoDB(artistid, extrasonly=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn('Cannot import blocked special purpose artist with id' + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.INCLUDE_EXTRAS, "Extras": headphones.EXTRAS } else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if artist and artist.get('artist_name') in blacklisted_special_artist_names: logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name')) myDB.action('DELETE from artists WHERE ArtistID=?', [artistid]) #in case it's already in the db myDB.action('DELETE from albums WHERE ArtistID=?', [artistid]) myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence over global option # Global options are set when adding a new artist myDB = db.DBConnection() try: db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False for rg in artist['releasegroups']: logger.info("Now adding/updating: " + rg['title']) rgid = rg['id'] # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() releases = mb.get_all_releases(rgid,includeExtras) if releases == []: logger.info('No official releases in release group %s' % rg['title']) continue if not releases: errors = True logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue # This will be used later to build a hybrid release fullreleaselist = [] for release in releases: # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table controlValueDict = {"ReleaseID" : release['ReleaseID']} newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "ReleaseDate": release['ReleaseDate'], "Type": release['Type'], "ReleaseCountry": release['ReleaseCountry'], "ReleaseFormat": release['ReleaseFormat'] } myDB.upsert("allalbums", newValueDict, controlValueDict) # Build the dictionary for the fullreleaselist newValueDict['ReleaseID'] = release['ReleaseID'] newValueDict['Tracks'] = release['Tracks'] fullreleaselist.append(newValueDict) for track in release['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": release['ReleaseID']} newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) except Exception, e: errors = True logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) if not rg_exists: releaseid = rg['id'] elif rg_exists and not rg_exists['ReleaseID']: # Need to do some importing here - to transition the old format of using the release group # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID # so we'll need to move over the locations, bitrates & formats from the tracks table to the new # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups logger.info("Copying current track information to alternate releases") tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() for track in tracks: if track['Location']: controlValueDict = {"TrackID": track['TrackID']} newValueDict = {"Location": track['Location'], "BitRate": track['BitRate'], "Format": track['Format'], } myDB.upsert("alltracks", newValueDict, controlValueDict) releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if not rg_exists: today = helpers.today() newValueDict['DateAdded']= today if headphones.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT total_track_count = len(tracks) for track in tracks: controlValueDict = {"TrackID": track['TrackID'], "AlbumID": rg['id']} newValueDict = {"ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) marked_as_downloaded = False if rg_exists: if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True else: if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True logger.info(u"Seeing if we need album art for " + rg['title']) cache.getThumb(AlbumID=rg['id']) #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: from headphones import searcher searcher.searchforalbum(albumid=rg['id'])
os.makedirs(self.path_to_art_cache) except Exception, e: logger.error('Unable to create artwork cache dir. Error: %s' + e) self.thumb_errors = True self.thumb_url = thumb_url #Delete the old stuff for thumb_file in self.thumb_files: try: os.remove(thumb_file) except: logger.error('Error deleting file from the cache: %s', thumb_file) ext = os.path.splitext(image_url)[1] thumb_path = os.path.join(self.path_to_art_cache, 'T_' + self.id + '.' + helpers.today() + ext) try: f = open(thumb_path, 'wb') f.write(artwork) f.close() except Exception, e: logger.error('Unable to write to the cache dir: %s', e) self.thumb_errors = True self.thumb_url = image_url def getArtwork(ArtistID=None, AlbumID=None): c = Cache() artwork_path = c.get_artwork_from_cache(ArtistID, AlbumID) if not artwork_path:
def home(self): page = [templates._header] if not headphones.CURRENT_VERSION: page.append('''<div class="updatebar">You're running an unknown version of Headphones. <a class="blue" href="update">Click here to update</a></div>''') elif headphones.CURRENT_VERSION != headphones.LATEST_VERSION and headphones.INSTALL_TYPE != 'win': page.append('''<div class="updatebar">A <a class="blue" href="http://github.com/rembo10/headphones/compare/%s...%s"> newer version</a> is available. You're %s commits behind. <a class="blue" href="update">Click here to update</a></div> ''' % (headphones.CURRENT_VERSION, headphones.LATEST_VERSION, headphones.COMMITS_BEHIND)) page.append(templates._logobar) page.append(templates._nav) myDB = db.DBConnection() results = myDB.select('SELECT ArtistName, ArtistID, Status, LatestAlbum, ReleaseDate, AlbumID, TotalTracks, HaveTracks from artists order by ArtistSortName collate nocase') if len(results): page.append('''<div class="table"><table border="0" cellpadding="3"> <tr> <th align="left" width="170">Artist Name</th> <th align="center" width="100">Status</th> <th align="center" width="300">Upcoming Albums</th> <th align="center">Have</th> </tr>''') for artist in results: totaltracks = artist['TotalTracks'] havetracks = artist['HaveTracks'] if not havetracks: havetracks = 0 try: percent = (havetracks*100.0)/totaltracks if percent > 100: percent = 100 except (ZeroDivisionError, TypeError): percent = 0 totaltracks = '?' if artist['LatestAlbum']: if artist['ReleaseDate'] > helpers.today(): newalbumName = '<a class="green" href="albumPage?AlbumID=%s"><i><b>%s</b></i>' % (artist['AlbumID'], artist['LatestAlbum']) releaseDate = '(%s)</a>' % artist['ReleaseDate'] else: newalbumName = '<a class="gray" href="albumPage?AlbumID=%s"><i>%s</i>' % (artist['AlbumID'], artist['LatestAlbum']) releaseDate = "" else: newalbumName = '<font color="#CFCFCF">None</font>' releaseDate = "" if artist['Status'] == 'Paused': newStatus = '''<font color="red"><b>%s</b></font>(<A class="external" href="resumeArtist?ArtistID=%s">resume</a>)''' % (artist['Status'], artist['ArtistID']) elif artist['Status'] == 'Loading': newStatus = '''<a class="gray">Loading...</a>''' else: newStatus = '''%s(<A class="external" href="pauseArtist?ArtistID=%s">pause</a>)''' % (artist['Status'], artist['ArtistID']) page.append('''<tr><td align="left" width="300"><a href="artistPage?ArtistID=%s">%s</a> (<A class="external" href="http://musicbrainz.org/artist/%s">link</a>) [<A class="externalred" href="deleteArtist?ArtistID=%s">delete</a>]</td> <td align="center" width="160">%s</td> <td align="center">%s %s</td> <td><div class="progress-container"><div style="width: %s%%"><div class="smalltext3">%s/%s</div></div></div></td></tr> ''' % (artist['ArtistID'], artist['ArtistName'], artist['ArtistID'], artist['ArtistID'], newStatus, newalbumName, releaseDate, percent, havetracks, totaltracks)) page.append('''</table></div>''') page.append(templates._footer % headphones.CURRENT_VERSION) else: have = myDB.select('SELECT ArtistName from have') if len(have): page.append("""<div class="datanil">Scanning...</div>""") else: page.append("""<div class="datanil">Add some artists to the database!</div>""") return page
def addArtisttoDB(artistid, extrasonly=False): # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return myDB = db.DBConnection() # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() if dbartist is None: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading"} else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} if headphones.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist['releasegroups']: rgid = rg['id'] # check if the album already exists rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rg['id']]) try: release_dict = mb.getReleaseGroup(rgid) except Exception, e: logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue if not release_dict: continue logger.info(u"Now adding/updating album: " + rg['title']) controlValueDict = {"AlbumID": rg['id']} if len(rg_exists): newValueDict = {"AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['releasedate'], } else: newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['releasedate'], "DateAdded": helpers.today(), "Type": rg['type'] } if release_dict['releasedate'] > helpers.today(): newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) try: lastfm.getAlbumDescription(rg['id'], artist['artist_name'], rg['title']) except Exception, e: logger.error('Attempt to retrieve album description from Last.fm failed: %s' % e)