def checkTorrentFinished(): """ Remove Torrent + data if Post Processed and finished Seeding """ logger.info( "Checking if any torrents have finished seeding and can be removed") myDB = db.DBConnection() results = myDB.select( 'SELECT * from snatched WHERE Status="Seed_Processed"') for album in results: hash = album['TorrentHash'] albumid = album['AlbumID'] torrent_removed = False if headphones.CONFIG.TORRENT_DOWNLOADER == 1: torrent_removed = transmission.removeTorrent(hash, True) elif headphones.CONFIG.TORRENT_DOWNLOADER == 2: torrent_removed = utorrent.removeTorrent(hash, True) elif headphones.CONFIG.TORRENT_DOWNLOADER == 3: torrent_removed = deluge.removeTorrent(hash, True) else: torrent_removed = qbittorrent.removeTorrent(hash, True) if torrent_removed: myDB.action( 'DELETE from snatched WHERE status = "Seed_Processed" and AlbumID=?', [albumid]) logger.info("Checking finished torrents completed")
def _queueAlbum(self, **kwargs): if 'id' not in kwargs: self.data = 'Missing parameter: id' return else: self.id = kwargs['id'] if 'new' in kwargs: new = kwargs['new'] else: new = False if 'lossless' in kwargs: lossless = kwargs['lossless'] else: lossless = False myDB = db.DBConnection() controlValueDict = {'AlbumID': self.id} if lossless: newValueDict = {'Status': 'Wanted Lossless'} else: newValueDict = {'Status': 'Wanted'} myDB.upsert("albums", newValueDict, controlValueDict) searcher.searchforalbum(self.id, new)
def updateFormat(): myDB = db.DBConnection() tracks = myDB.select('SELECT * from tracks WHERE Location IS NOT NULL and Format IS NULL') if len(tracks) > 0: logger.info('Finding media format for %s files' % len(tracks)) for track in tracks: try: f = MediaFile(track['Location']) except Exception as e: logger.info("Exception from MediaFile for: " + track['Location'] + " : " + str(e)) continue controlValueDict = {"TrackID": track['TrackID']} newValueDict = {"Format": f.format} myDB.upsert("tracks", newValueDict, controlValueDict) logger.info('Finished finding media format for %s files' % len(tracks)) havetracks = myDB.select('SELECT * from have WHERE Location IS NOT NULL and Format IS NULL') if len(havetracks) > 0: logger.info('Finding media format for %s files' % len(havetracks)) for track in havetracks: try: f = MediaFile(track['Location']) except Exception as e: logger.info("Exception from MediaFile for: " + track['Location'] + " : " + str(e)) continue controlValueDict = {"TrackID": track['TrackID']} newValueDict = {"Format": f.format} myDB.upsert("have", newValueDict, controlValueDict) logger.info('Finished finding media format for %s files' % len(havetracks))
def getArtists(): myDB = db.DBConnection() results = myDB.select("SELECT ArtistID from artists") if not headphones.CONFIG.LASTFM_USERNAME: logger.warn("Last.FM username not set, not importing artists.") return logger.info("Fetching artists from Last.FM for username: %s", headphones.CONFIG.LASTFM_USERNAME) data = request_lastfm("library.getartists", limit=1000, user=headphones.CONFIG.LASTFM_USERNAME) if data and "artists" in data: artistlist = [] artists = data["artists"]["artist"] logger.debug("Fetched %d artists from Last.FM", len(artists)) for artist in artists: artist_mbid = artist["mbid"] if not any(artist_mbid in x for x in results): artistlist.append(artist_mbid) from headphones import importer for artistid in artistlist: importer.addArtisttoDB(artistid) logger.info("Imported %d new artists from Last.FM", len(artistlist))
def get_info_from_cache(self, ArtistID=None, AlbumID=None): self.query_type = 'info' myDB = db.DBConnection() if ArtistID: self.id = ArtistID self.id_type = 'artist' db_info = myDB.action( 'SELECT Summary, Content, LastUpdated FROM descriptions WHERE ArtistID=?', [self.id]).fetchone() else: self.id = AlbumID self.id_type = 'album' db_info = myDB.action( 'SELECT Summary, Content, LastUpdated FROM descriptions WHERE ReleaseGroupID=?', [self.id]).fetchone() if not db_info or not db_info['LastUpdated'] or not self._is_current( date=db_info['LastUpdated']): self._update_cache() info_dict = {'Summary': self.info_summary, 'Content': self.info_content} return info_dict else: info_dict = {'Summary': db_info['Summary'], 'Content': db_info['Content']} return info_dict
def _delArtist(self, **kwargs): if 'id' not in kwargs: self.data = 'Missing parameter: id' return else: self.id = kwargs['id'] myDB = db.DBConnection() myDB.action('DELETE from artists WHERE ArtistID="' + self.id + '"') myDB.action('DELETE from albums WHERE ArtistID="' + self.id + '"') myDB.action('DELETE from tracks WHERE ArtistID="' + self.id + '"')
def _resumeArtist(self, **kwargs): if 'id' not in kwargs: self.data = 'Missing parameter: id' return else: self.id = kwargs['id'] myDB = db.DBConnection() controlValueDict = {'ArtistID': self.id} newValueDict = {'Status': 'Active'} myDB.upsert("artists", newValueDict, controlValueDict)
def _dic_from_query(self, query): myDB = db.DBConnection() rows = myDB.select(query) rows_as_dic = [] for row in rows: row_as_dic = dict(zip(row.keys(), row)) rows_as_dic.append(row_as_dic) return rows_as_dic
def _unqueueAlbum(self, **kwargs): if 'id' not in kwargs: self.data = 'Missing parameter: id' return else: self.id = kwargs['id'] myDB = db.DBConnection() controlValueDict = {'AlbumID': self.id} newValueDict = {'Status': 'Skipped'} myDB.upsert("albums", newValueDict, controlValueDict)
def is_exists(artistid): myDB = db.DBConnection() # See if the artist is already in the database artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', [artistid]) if any(artistid in x for x in artistlist): logger.info(artistlist[0][ 1] + u" is already in the database. Updating 'have tracks', but not artist information") return True else: return False
def dbUpdate(forcefull=False): myDB = db.DBConnection() active_artists = myDB.select( 'SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC' ) logger.info('Starting update for %i active artists', len(active_artists)) for artist in active_artists: artistid = artist[0] importer.addArtisttoDB(artistid=artistid, extrasonly=False, forcefull=forcefull) logger.info('Active artist update complete')
def update_album_status(AlbumID=None): myDB = db.DBConnection() logger.info('Counting matched tracks to mark albums as skipped/downloaded') if AlbumID: album_status_updater = myDB.action( 'SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID]) else: album_status_updater = myDB.action( 'SELECT AlbumID, AlbumTitle, Status from albums') for album in album_status_updater: track_counter = myDB.action( 'SELECT Location from tracks where AlbumID=?', [album['AlbumID']]) total_tracks = 0 have_tracks = 0 for track in track_counter: total_tracks += 1 if track['Location']: have_tracks += 1 if total_tracks != 0: album_completion = float( float(have_tracks) / float(total_tracks)) * 100 else: album_completion = 0 logger.info('Album %s does not have any tracks in database' % album['AlbumTitle']) if album_completion >= headphones.CONFIG.ALBUM_COMPLETION_PCT: new_album_status = "Downloaded" # I don't think we want to change Downloaded->Skipped..... # I think we can only automatically change Skipped->Downloaded when updating # There was a bug report where this was causing infinite downloads if the album was # recent, but matched to less than 80%. It would go Downloaded->Skipped->Wanted->Downloaded->Skipped->Wanted->etc.... # else: # if album['Status'] == "Skipped" or album['Status'] == "Downloaded": # new_album_status = "Skipped" # else: # new_album_status = album['Status'] else: new_album_status = album['Status'] myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']}) if new_album_status != album['Status']: logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) logger.info('Album status update complete')
def findArtistbyAlbum(name): myDB = db.DBConnection() artist = myDB.action( 'SELECT AlbumTitle from have WHERE ArtistName=? AND AlbumTitle IS NOT NULL ORDER BY RANDOM()', [name]).fetchone() if not artist: return False # Probably not neccessary but just want to double check if not artist['AlbumTitle']: return False term = '"' + artist['AlbumTitle'] + '" AND artist:"' + name + '"' results = None try: with mb_lock: results = musicbrainzngs.search_release_groups(term).get( 'release-group-list') except musicbrainzngs.WebServiceError as e: logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e))) mb_lock.snooze(5) if not results: return False artist_dict = {} for releaseGroup in results: newArtist = releaseGroup['artist-credit'][0]['artist'] # Only need the artist ID if we're doing an artist+album lookup # if 'disambiguation' in newArtist: # uniquename = unicode(newArtist['sort-name'] + " (" + newArtist['disambiguation'] + ")") # else: # uniquename = unicode(newArtist['sort-name']) # artist_dict['name'] = unicode(newArtist['sort-name']) # artist_dict['uniquename'] = uniquename artist_dict['id'] = unicode(newArtist['id']) # artist_dict['url'] = u'http://musicbrainz.org/artist/' + newArtist['id'] # artist_dict['score'] = int(releaseGroup['ext:score']) return artist_dict
def getSimilar(): myDB = db.DBConnection() results = myDB.select( "SELECT ArtistID from artists ORDER BY HaveTracks DESC") logger.info("Fetching similar artists from Last.FM for tag cloud") artistlist = [] for result in results[:12]: data = request_lastfm("artist.getsimilar", mbid=result["ArtistId"]) if data and "similarartists" in data: artists = data["similarartists"]["artist"] for artist in artists: try: artist_mbid = artist["mbid"] artist_name = artist["name"] except KeyError: continue if not any(artist_mbid in x for x in results): artistlist.append((artist_name, artist_mbid)) # Add new artists to tag cloud logger.debug("Fetched %d artists from Last.FM", len(artistlist)) count = defaultdict(int) for artist, mbid in artistlist: count[artist, mbid] += 1 items = count.items() top_list = sorted(items, key=lambda x: x[1], reverse=True)[:25] random.shuffle(top_list) myDB.action("DELETE from lastfmcloud") for item in top_list: artist_name, artist_mbid = item[0] count = item[1] myDB.action("INSERT INTO lastfmcloud VALUES( ?, ?, ?)", [artist_name, artist_mbid, count]) logger.debug("Inserted %d artists into Last.FM tag cloud", len(top_list))
def _download_specific_release(self, **kwargs): expected_kwargs = ['id', 'title', 'size', 'url', 'provider', 'kind'] for kwarg in expected_kwargs: if kwarg not in kwargs: self.data = 'Missing parameter: ' + kwarg return self.data title = kwargs['title'] size = kwargs['size'] url = kwargs['url'] provider = kwargs['provider'] kind = kwargs['kind'] id = kwargs['id'] for kwarg in expected_kwargs: del kwargs[kwarg] # Handle situations where the torrent url contains arguments that are # parsed if kwargs: import urllib import urllib2 url = urllib2.quote(url, safe=":?/=&") + '&' + urllib.urlencode(kwargs) try: result = [(title, int(size), url, provider, kind)] except ValueError: result = [(title, float(size), url, provider, kind)] logger.info(u"Making sure we can download the chosen result") (data, bestqual) = searcher.preprocess(result) if data and bestqual: myDB = db.DBConnection() album = myDB.action('SELECT * from albums WHERE AlbumID=?', [id]).fetchone() searcher.send_to_downloader(data, bestqual, album)
def finalize_update(artistid, artistname, errors=False): # Moving this little bit to it's own function so we can update have tracks & latest album when deleting extras myDB = db.DBConnection() latestalbum = myDB.action( 'SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")', [artistid])) # havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) havetracks = len( myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len( myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artistname])) controlValueDict = {"ArtistID": artistid} if latestalbum: newValueDict = {"Status": "Active", "LatestAlbum": latestalbum['AlbumTitle'], "ReleaseDate": latestalbum['ReleaseDate'], "AlbumID": latestalbum['AlbumID'], "TotalTracks": totaltracks, "HaveTracks": havetracks} else: newValueDict = {"Status": "Active", "TotalTracks": totaltracks, "HaveTracks": havetracks} if not errors: newValueDict['LastUpdated'] = helpers.now() myDB.upsert("artists", newValueDict, controlValueDict)
def getTagTopArtists(tag, limit=50): myDB = db.DBConnection() results = myDB.select("SELECT ArtistID from artists") logger.info("Fetching top artists from Last.FM for tag: %s", tag) data = request_lastfm("tag.gettopartists", limit=limit, tag=tag) if data and "topartists" in data: artistlist = [] artists = data["topartists"]["artist"] logger.debug("Fetched %d artists from Last.FM", len(artists)) for artist in artists: artist_mbid = artist["mbid"] if not any(artist_mbid in x for x in results): artistlist.append(artist_mbid) from headphones import importer for artistid in artistlist: importer.addArtisttoDB(artistid) logger.debug("Added %d new artists from Last.FM", len(artistlist))
def get_new_releases(rgid, includeExtras=False, forcefull=False): myDB = db.DBConnection() results = [] release_status = "official" if includeExtras and not headphones.CONFIG.OFFICIAL_RELEASES_ONLY: release_status = [] try: limit = 100 newResults = None while newResults is None or len(newResults) >= limit: with mb_lock: newResults = musicbrainzngs.browse_releases( release_group=rgid, includes=[ 'artist-credits', 'labels', 'recordings', 'release-groups', 'media' ], release_status=release_status, limit=limit, offset=len(results)) if 'release-list' not in newResults: break # may want to raise an exception here instead ? newResults = newResults['release-list'] results += newResults except musicbrainzngs.WebServiceError as e: logger.warn( 'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) mb_lock.snooze(5) return False if not results or len(results) == 0: return False # Clean all references to releases in dB that are no longer referenced in musicbrainz release_list = [] force_repackage1 = 0 if len(results) != 0: for release_mark in results: release_list.append(unicode(release_mark['id'])) release_title = release_mark['title'] remove_missing_releases = myDB.action( "SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) if remove_missing_releases: for items in remove_missing_releases: if items['ReleaseID'] not in release_list and items[ 'ReleaseID'] != rgid: # Remove all from albums/tracks that aren't in release myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) logger.info( "Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) force_repackage1 = 1 else: logger.info( "There was either an error pulling data from MusicBrainz or there might not be any releases for this category" ) num_new_releases = 0 for releasedata in results: release = {} rel_id_check = releasedata['id'] album_checker = myDB.action( 'SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() if not album_checker or forcefull: # DELETE all references to this release since we're updating it anyway. myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check]) myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check]) release['AlbumTitle'] = unicode(releasedata['title']) release['AlbumID'] = unicode(rgid) release['AlbumASIN'] = unicode( releasedata['asin']) if 'asin' in releasedata else None release['ReleaseDate'] = unicode( releasedata['date']) if 'date' in releasedata else None release['ReleaseID'] = releasedata['id'] if 'release-group' not in releasedata: raise Exception( 'No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) release['Type'] = unicode(releasedata['release-group']['type']) if release[ 'Type'] == 'Album' and 'secondary-type-list' in releasedata[ 'release-group']: secondary_type = unicode( releasedata['release-group']['secondary-type-list'][0]) if secondary_type != release['Type']: release['Type'] = secondary_type # making the assumption that the most important artist will be first in the list if 'artist-credit' in releasedata: release['ArtistID'] = unicode( releasedata['artist-credit'][0]['artist']['id']) release['ArtistName'] = unicode( releasedata['artist-credit-phrase']) else: logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') return False release['ReleaseCountry'] = unicode( releasedata['country'] ) if 'country' in releasedata else u'Unknown' # assuming that the list will contain media and that the format will be consistent try: additional_medium = '' for position in releasedata['medium-list']: if position['format'] == releasedata['medium-list'][0][ 'format']: medium_count = int(position['position']) else: additional_medium = additional_medium + ' + ' + position[ 'format'] if medium_count == 1: disc_number = '' else: disc_number = str(medium_count) + 'x' packaged_medium = disc_number + releasedata['medium-list'][0][ 'format'] + additional_medium release['ReleaseFormat'] = unicode(packaged_medium) except: release['ReleaseFormat'] = u'Unknown' release['Tracks'] = getTracksFromRelease(releasedata) # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table controlValueDict = {"ReleaseID": release['ReleaseID']} newValueDict = { "ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "ReleaseDate": release['ReleaseDate'], "Type": release['Type'], "ReleaseCountry": release['ReleaseCountry'], "ReleaseFormat": release['ReleaseFormat'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in release['Tracks']: cleanname = helpers.clean_name(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) controlValueDict = { "TrackID": track['id'], "ReleaseID": release['ReleaseID'] } newValueDict = { "ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [ release['ArtistName'], release['AlbumTitle'], track['title'] ]).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) num_new_releases = num_new_releases + 1 if album_checker: logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check)) else: logger.info('[%s] New release %s (%s) added' % (release['ArtistName'], release['AlbumTitle'], rel_id_check)) if force_repackage1 == 1: num_new_releases = -1 logger.info( '[%s] Forcing repackage of %s, since dB releases have been removed' % (release['ArtistName'], release_title)) else: num_new_releases = num_new_releases return num_new_releases
def getArtist(artistid, extrasonly=False): artist_dict = {} artist = None try: limit = 100 with mb_lock: artist = musicbrainzngs.get_artist_by_id(artistid)['artist'] newRgs = None artist['release-group-list'] = [] while newRgs is None or len(newRgs) >= limit: with mb_lock: newRgs = musicbrainzngs.browse_release_groups( artistid, release_type="album", offset=len(artist['release-group-list']), limit=limit) newRgs = newRgs['release-group-list'] artist['release-group-list'] += newRgs except musicbrainzngs.WebServiceError as e: logger.warn( 'Attempt to retrieve artist information from MusicBrainz failed for artistid: %s (%s)' % (artistid, str(e))) mb_lock.snooze(5) except Exception as e: pass if not artist: return False artist_dict['artist_name'] = unicode(artist['name']) releasegroups = [] if not extrasonly: for rg in artist['release-group-list']: if "secondary-type-list" in rg.keys( ): # only add releases without a secondary type continue releasegroups.append({ 'title': unicode(rg['title']), 'id': unicode(rg['id']), 'url': u"http://musicbrainz.org/release-group/" + rg['id'], 'type': unicode(rg['type']) }) # See if we need to grab extras. Artist specific extras take precedence over global option # Global options are set when adding a new artist myDB = db.DBConnection() try: db_artist = myDB.action( 'SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False if includeExtras: # Need to convert extras string from something like '2,5.6' to ['ep','live','remix'] (append new extras to end) if db_artist['Extras']: extras = map(int, db_artist['Extras'].split(',')) else: extras = [] extras_list = headphones.POSSIBLE_EXTRAS includes = [] i = 1 for extra in extras_list: if i in extras: includes.append(extra) i += 1 for include in includes: mb_extras_list = [] try: limit = 100 newRgs = None while newRgs is None or len(newRgs) >= limit: with mb_lock: newRgs = musicbrainzngs.browse_release_groups( artistid, release_type=include, offset=len(mb_extras_list), limit=limit) newRgs = newRgs['release-group-list'] mb_extras_list += newRgs except musicbrainzngs.WebServiceError as e: logger.warn( 'Attempt to retrieve artist information from MusicBrainz failed for artistid: %s (%s)' % (artistid, str(e))) mb_lock.snooze(5) for rg in mb_extras_list: rg_type = rg['type'] if rg_type == 'Album' and 'secondary-type-list' in rg: secondary_type = rg['secondary-type-list'][0] if secondary_type != rg_type: rg_type = secondary_type releasegroups.append({ 'title': unicode(rg['title']), 'id': unicode(rg['id']), 'url': u"http://musicbrainz.org/release-group/" + rg['id'], 'type': unicode(rg_type) }) artist_dict['releasegroups'] = releasegroups return artist_dict
def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn('Cannot import blocked special purpose artist with id' + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS, "Extras": headphones.CONFIG.EXTRAS} if type == "series": newValueDict['Type'] = "series" else: newValueDict = {"Status": "Loading"} if dbartist["Type"] == "series": type = "series" myDB.upsert("artists", newValueDict, controlValueDict) if type == "series": artist = mb.getSeries(artistid) else: artist = mb.getArtist(artistid, extrasonly) if artist and artist.get('artist_name') in blacklisted_special_artist_names: logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name')) myDB.action('DELETE from artists WHERE ArtistID=?', [artistid]) # in case it's already in the db myDB.action('DELETE from albums WHERE ArtistID=?', [artistid]) myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence # over global option. Global options are set when adding a new artist try: db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False # Clean all references to release group in dB that are no longer referenced # from the musicbrainz refresh group_list = [] force_repackage = 0 # Don't nuke the database if there's a MusicBrainz error if len(artist['releasegroups']) != 0: for groups in artist['releasegroups']: group_list.append(groups['id']) if not extrasonly: remove_missing_groups_from_albums = myDB.select( "SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) else: remove_missing_groups_from_albums = myDB.select( 'SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid]) for items in remove_missing_groups_from_albums: if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']]) logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % ( artist['artist_name'], items['AlbumID'])) if not extrasonly: force_repackage = 1 else: if not extrasonly: logger.info( "[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name']) # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks album_searches = [] for rg in artist['releasegroups']: al_title = rg['title'] today = helpers.today() rgid = rg['id'] skip_log = 0 # Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.CONFIG.MB_IGNORE_AGE rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() if not forcefull: new_release_group = False try: check_release_date = rg_exists['ReleaseDate'] except TypeError: check_release_date = None new_release_group = True if new_release_group: logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid, includeExtras) else: if check_release_date is None or check_release_date == u"None": logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: if len(check_release_date) == 10: release_date = check_release_date elif len(check_release_date) == 7: release_date = check_release_date + "-31" elif len(check_release_date) == 4: release_date = check_release_date + "-12-31" else: release_date = today if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta: logger.info("[%s] Now updating: %s (Release Date <%s Days)", artist['artist_name'], rg['title'], pause_delta) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: logger.info("[%s] Skipping: %s (Release Date >%s Days)", artist['artist_name'], rg['title'], pause_delta) skip_log = 1 new_releases = 0 if force_repackage == 1: new_releases = -1 logger.info('[%s] Forcing repackage of %s (Release Group Removed)', artist['artist_name'], al_title) else: new_releases = new_releases else: logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'], rg['title']) new_releases = mb.get_new_releases(rgid, includeExtras, forcefull) if new_releases != 0: # Dump existing hybrid release since we're repackaging/replacing it myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rg['id']]) # This will be used later to build a hybrid release fullreleaselist = [] # Search for releases within a release group find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) # Build the dictionary for the fullreleaselist for items in find_hybrid_releases: # don't include hybrid information, since that's what we're replacing if items['ReleaseID'] != rg['id']: hybrid_release_id = items['ReleaseID'] newValueDict = {"ArtistID": items['ArtistID'], "ArtistName": items['ArtistName'], "AlbumTitle": items['AlbumTitle'], "AlbumID": items['AlbumID'], "AlbumASIN": items['AlbumASIN'], "ReleaseDate": items['ReleaseDate'], "Type": items['Type'], "ReleaseCountry": items['ReleaseCountry'], "ReleaseFormat": items['ReleaseFormat'] } find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] for hybrid_tracks in find_hybrid_tracks: hybrid_track_array.append({ 'number': hybrid_tracks['TrackNumber'], 'title': hybrid_tracks['TrackTitle'], 'id': hybrid_tracks['TrackID'], # 'url': hybrid_tracks['TrackURL'], 'duration': hybrid_tracks['TrackDuration'] }) totalTracks += 1 newValueDict['ReleaseID'] = hybrid_release_id newValueDict['Tracks'] = hybrid_track_array fullreleaselist.append(newValueDict) # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) logger.info('[%s] Packaging %s releases into hybrid title' % ( artist['artist_name'], rg['title'])) except Exception as e: errors = True logger.warn('[%s] Unable to get hybrid release information for %s: %s' % ( artist['artist_name'], rg['title'], e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.clean_name(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table # myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) # check if the album already exists if not rg_exists: releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] if not releaseid: releaseid = rg['id'] album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if rg_exists: newValueDict['DateAdded'] = rg_exists['DateAdded'] newValueDict['Status'] = rg_exists['Status'] else: today = helpers.today() newValueDict['DateAdded'] = today if headphones.CONFIG.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif helpers.get_age(album['ReleaseDate']) and helpers.get_age( today) - helpers.get_age( album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to # mark it as downloaded. Default is 80%, can be set in config as # ALBUM_COMPLETION_PCT total_track_count = len(tracks) if total_track_count == 0: logger.warning("Total track count is zero for Release ID " + "'%s', skipping.", releaseid) continue for track in tracks: controlValueDict = {"TrackID": track['TrackID'], "AlbumID": rg['id']} newValueDict = {"ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len( myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) marked_as_downloaded = False if rg_exists: if rg_exists['Status'] == 'Skipped' and ( (have_track_count / float(total_track_count)) >= ( headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True else: if (have_track_count / float(total_track_count)) >= ( headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True logger.info( u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title'])) cache.getThumb(AlbumID=rg['id']) # Start a search for the album if it's new, hasn't been marked as # downloaded and autowant_all is selected. This search is deferred, # in case the search failes and the rest of the import will halt. if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL: album_searches.append(rg['id']) else: if skip_log == 0: logger.info(u"[%s] No new releases, so no changes made to %s" % ( artist['artist_name'], rg['title'])) time.sleep(3) finalize_update(artistid, artist['artist_name'], errors) logger.info(u"Seeing if we need album art for: %s" % artist['artist_name']) cache.getThumb(ArtistID=artistid) logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name']) metacritic.update(artistid, artist['artist_name'], artist['releasegroups']) if errors: logger.info( "[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % ( artist['artist_name'], artist['artist_name'])) else: myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']]) logger.info(u"Updating complete for: %s" % artist['artist_name']) # Start searching for newly added albums if album_searches: from headphones import searcher logger.info("Start searching for %d albums.", len(album_searches)) for album_search in album_searches: searcher.searchforalbum(albumid=album_search)
def artistlist_to_mbids(artistlist, forced=False): for artist in artistlist: if not artist and artist != ' ': continue # If adding artists through Manage New Artists, they're coming through as non-unicode (utf-8?) # and screwing everything up if not isinstance(artist, unicode): try: artist = artist.decode('utf-8', 'replace') except Exception: logger.warn("Unable to convert artist to unicode so cannot do a database lookup") continue results = mb.findArtist(artist, limit=1) if not results: logger.info('No results found for: %s' % artist) continue try: artistid = results[0]['id'] except IndexError: logger.info('MusicBrainz query turned up no matches for: %s' % artist) continue # Check if it's blacklisted/various artists (only check if it's not forced, e.g. through library scan auto-add.) # Forced example = Adding an artist from Manage New Artists myDB = db.DBConnection() if not forced: bl_artist = myDB.action('SELECT * FROM blacklist WHERE ArtistID=?', [artistid]).fetchone() if bl_artist or artistid in blacklisted_special_artists: logger.info("Artist ID for '%s' is either blacklisted or Various Artists. To add artist, you must " "do it manually (Artist ID: %s)" % (artist, artistid)) continue # Add to database if it doesn't exist if not is_exists(artistid): addArtisttoDB(artistid) # Just update the tracks if it does else: havetracks = len( myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len( myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid]) # Delete it from the New Artists if the request came from there if forced: myDB.action('DELETE from newartists WHERE ArtistName=?', [artist]) # Update the similar artist tag cloud: logger.info('Updating artist information from Last.fm') try: lastfm.getSimilar() except Exception as e: logger.warn('Failed to update arist information from Last.fm: %s' % e)
def addReleaseById(rid, rgid=None): myDB = db.DBConnection() # Create minimum info upfront if added from searchresults status = '' if rgid: dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) if not dbalbum: status = 'Loading' controlValueDict = {"AlbumID": rgid} newValueDict = {"AlbumTitle": rgid, "ArtistName": status, "Status": status} myDB.upsert("albums", newValueDict, controlValueDict) time.sleep(1) rgid = None artistid = None release_dict = None results = myDB.select( "SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid]) for result in results: rgid = result['ReleaseGroupID'] artistid = result['ArtistID'] logger.debug( "Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid) if not rgid: # didn't find it in the cache, get the information from MB logger.debug( "Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID") try: release_dict = mb.getRelease(rid) except Exception as e: logger.info('Unable to get release information for Release %s: %s', rid, e) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not release_dict: logger.info('Unable to get release information for Release %s: no dict', rid) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return rgid = release_dict['rgid'] artistid = release_dict['artist_id'] # we don't want to make more calls to MB here unless we have to, could be happening quite a lot rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) # make sure the artist exists since I don't know what happens later if it doesn't artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) if not artist_exists and release_dict: if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: sortname = release_dict['artist_name'] logger.info( u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused") controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = {"ArtistName": release_dict['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused"} if headphones.CONFIG.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 newValueDict['Extras'] = headphones.CONFIG.EXTRAS if 'title' in release_dict: newValueDict['LatestAlbum'] = release_dict['title'] elif 'rg_title' in release_dict: newValueDict['LatestAlbum'] = release_dict['rg_title'] if 'date' in release_dict: newValueDict['ReleaseDate'] = release_dict['date'] if rgid: newValueDict['AlbumID'] = rgid myDB.upsert("artists", newValueDict, controlValueDict) elif not artist_exists and not release_dict: logger.error( "Artist does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not rg_exists and release_dict or status == 'Loading' and release_dict: # it should never be the case that we have an rg and not the artist # but if it is this will fail logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid) controlValueDict = {"AlbumID": rgid} if status != 'Loading': status = 'Wanted' newValueDict = {"ArtistID": release_dict['artist_id'], "ReleaseID": rgid, "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'] if 'title' in release_dict else release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['date'], "DateAdded": helpers.today(), "Status": status, "Type": release_dict['rg_type'], "ReleaseID": rid } myDB.upsert("albums", newValueDict, controlValueDict) # keep a local cache of these so that external programs that are adding releasesByID don't hammer MB myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: cleanname = helpers.clean_name( release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "AlbumID": rgid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action( 'SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] # myDB.action('DELETE from have WHERE Location=?', [match['Location']]) # If the album has been scanned before adding the release it will be unmatched, update to matched if match['Matched'] == 'Failed': myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release_dict['rgid'], match['Location'])) myDB.upsert("tracks", newValueDict, controlValueDict) # Reset status if status == 'Loading': controlValueDict = {"AlbumID": rgid} if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: newValueDict = {"Status": "Wanted"} else: newValueDict = {"Status": "Skipped"} myDB.upsert("albums", newValueDict, controlValueDict) # Start a search for the album if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: import searcher searcher.searchforalbum(rgid, False) elif not rg_exists and not release_dict: logger.error( "ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return else: logger.info('Release ' + str(rid) + " already exists in the database!")
def switch(AlbumID, ReleaseID): """ Takes the contents from allalbums & alltracks (based on ReleaseID) and switches them into the albums & tracks table. """ logger.debug('Switching allalbums and alltracks') myDB = db.DBConnection() oldalbumdata = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() newalbumdata = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [ReleaseID]).fetchone() newtrackdata = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [ReleaseID]).fetchall() myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID]) controlValueDict = {"AlbumID": AlbumID} newValueDict = { "ArtistID": newalbumdata['ArtistID'], "ArtistName": newalbumdata['ArtistName'], "AlbumTitle": newalbumdata['AlbumTitle'], "ReleaseID": newalbumdata['ReleaseID'], "AlbumASIN": newalbumdata['AlbumASIN'], "ReleaseDate": newalbumdata['ReleaseDate'], "Type": newalbumdata['Type'], "ReleaseCountry": newalbumdata['ReleaseCountry'], "ReleaseFormat": newalbumdata['ReleaseFormat'] } myDB.upsert("albums", newValueDict, controlValueDict) # Update cache c = cache.Cache() c.remove_from_cache(AlbumID=AlbumID) c.get_artwork_from_cache(AlbumID=AlbumID) for track in newtrackdata: controlValueDict = {"TrackID": track['TrackID'], "AlbumID": AlbumID} newValueDict = { "ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, # configurable) of the album total_track_count = len(newtrackdata) have_track_count = len( myDB.select( 'SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [AlbumID])) if oldalbumdata['Status'] == 'Skipped' and ( (have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', AlbumID]) # Update have track counts on index totaltracks = len( myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")', [newalbumdata['ArtistID']])) havetracks = len( myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [newalbumdata['ArtistID']])) controlValueDict = {"ArtistID": newalbumdata['ArtistID']} newValueDict = {"TotalTracks": totaltracks, "HaveTracks": havetracks} myDB.upsert("artists", newValueDict, controlValueDict)
def update(artistid, artist_name, release_groups): """ Pretty simple and crude function to find the artist page on metacritic, then parse that page to get critic & user scores for albums""" # First let's modify the artist name to fit the metacritic convention. # We could just do a search, then take the top result, but at least this will # cut down on api calls. If it's ineffective then we'll switch to search replacements = {" & ": " ", ".": ""} mc_artist_name = helpers.replace_all(artist_name.lower(), replacements) mc_artist_name = mc_artist_name.replace(" ", "-") headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' } url = "http://www.metacritic.com/person/" + mc_artist_name + "?filter-options=music&sort_options=date&num_items=100" res = request.request_soup(url, headers=headers) rows = None try: table = res.find("table", class_="credits person_credits") rows = table.tbody.find_all('tr') except: logger.info("Unable to get metacritic scores for: %s" % artist_name) myDB = db.DBConnection() artist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() score_list = [] # If we couldn't get anything from MetaCritic for whatever reason, # let's try to load scores from the db if not rows: if artist['MetaCritic']: score_list = json.loads(artist['MetaCritic']) else: return # If we did get scores, let's update the db with them else: for row in rows: title = row.a.string scores = row.find_all("span") critic_score = scores[0].string user_score = scores[1].string score_dict = { 'title': title, 'critic_score': critic_score, 'user_score': user_score } score_list.append(score_dict) # Save scores to the database controlValueDict = {"ArtistID": artistid} newValueDict = {'MetaCritic': json.dumps(score_list)} myDB.upsert("artists", newValueDict, controlValueDict) for score in score_list: title = score['title'] # Iterate through the release groups we got passed to see if we can find # a match for rg in release_groups: if rg['title'].lower() == title.lower(): critic_score = score['critic_score'] user_score = score['user_score'] controlValueDict = {"AlbumID": rg['id']} newValueDict = { 'CriticScore': critic_score, 'UserScore': user_score } myDB.upsert("albums", newValueDict, controlValueDict)
def _update_cache(self): """ Since we call the same url for both info and artwork, we'll update both at the same time """ myDB = db.DBConnection() # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums # Exception is when adding albums manually, then we should use release id if self.id_type == 'artist': data = lastfm.request_lastfm("artist.getinfo", mbid=self.id, api_key=LASTFM_API_KEY) if not data: return try: self.info_summary = data['artist']['bio']['summary'] except KeyError: logger.debug('No artist bio summary found') self.info_summary = None try: self.info_content = data['artist']['bio']['content'] except KeyError: logger.debug('No artist bio found') self.info_content = None try: image_url = data['artist']['image'][-1]['#text'] except KeyError: logger.debug('No artist image found') image_url = None thumb_url = self._get_thumb_url(data) if not thumb_url: logger.debug('No artist thumbnail image found') else: dbalbum = myDB.action( 'SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?', [self.id]).fetchone() if dbalbum['ReleaseID'] != self.id: data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'], api_key=LASTFM_API_KEY) if not data: data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY) else: data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY) if not data: return try: self.info_summary = data['album']['wiki']['summary'] except KeyError: logger.debug('No album summary found') self.info_summary = None try: self.info_content = data['album']['wiki']['content'] except KeyError: logger.debug('No album infomation found') self.info_content = None try: image_url = data['album']['image'][-1]['#text'] except KeyError: logger.debug('No album image link found') image_url = None thumb_url = self._get_thumb_url(data) if not thumb_url: logger.debug('No album thumbnail image found') # Save the content & summary to the database no matter what if we've # opened up the url if self.id_type == 'artist': controlValueDict = {"ArtistID": self.id} else: controlValueDict = {"ReleaseGroupID": self.id} newValueDict = {"Summary": self.info_summary, "Content": self.info_content, "LastUpdated": helpers.today()} myDB.upsert("descriptions", newValueDict, controlValueDict) # Save the image URL to the database if image_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ArtworkURL=? WHERE ArtistID=?', [image_url, self.id]) else: myDB.action('UPDATE albums SET ArtworkURL=? WHERE AlbumID=?', [image_url, self.id]) # Save the thumb URL to the database if thumb_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ThumbURL=? WHERE ArtistID=?', [thumb_url, self.id]) else: myDB.action('UPDATE albums SET ThumbURL=? WHERE AlbumID=?', [thumb_url, self.id]) # Should we grab the artwork here if we're just grabbing thumbs or # info? Probably not since the files can be quite big if image_url and self.query_type == 'artwork': artwork = request.request_content(image_url, timeout=20) if artwork: # Make sure the artwork dir exists: if not os.path.isdir(self.path_to_art_cache): try: os.makedirs(self.path_to_art_cache) os.chmod(self.path_to_art_cache, int(headphones.CONFIG.FOLDER_PERMISSIONS, 8)) except OSError as e: logger.error('Unable to create artwork cache dir. Error: %s', e) self.artwork_errors = True self.artwork_url = image_url # Delete the old stuff for artwork_file in self.artwork_files: try: os.remove(artwork_file) except: logger.error('Error deleting file from the cache: %s', artwork_file) ext = os.path.splitext(image_url)[1] artwork_path = os.path.join(self.path_to_art_cache, self.id + '.' + helpers.today() + ext) try: with open(artwork_path, 'wb') as f: f.write(artwork) os.chmod(artwork_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8)) except (OSError, IOError) as e: logger.error('Unable to write to the cache dir: %s', e) self.artwork_errors = True self.artwork_url = image_url # Grab the thumbnail as well if we're getting the full artwork (as long # as it's missing/outdated. if thumb_url and self.query_type in ['thumb', 'artwork'] and not ( self.thumb_files and self._is_current(self.thumb_files[0])): artwork = request.request_content(thumb_url, timeout=20) if artwork: # Make sure the artwork dir exists: if not os.path.isdir(self.path_to_art_cache): try: os.makedirs(self.path_to_art_cache) os.chmod(self.path_to_art_cache, int(headphones.CONFIG.FOLDER_PERMISSIONS, 8)) except OSError as e: logger.error('Unable to create artwork cache dir. Error: %s' + e) self.thumb_errors = True self.thumb_url = thumb_url # Delete the old stuff for thumb_file in self.thumb_files: try: os.remove(thumb_file) except OSError as e: logger.error('Error deleting file from the cache: %s', thumb_file) ext = os.path.splitext(image_url)[1] thumb_path = os.path.join(self.path_to_art_cache, 'T_' + self.id + '.' + helpers.today() + ext) try: with open(thumb_path, 'wb') as f: f.write(artwork) os.chmod(thumb_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8)) except (OSError, IOError) as e: logger.error('Unable to write to the cache dir: %s', e) self.thumb_errors = True self.thumb_url = image_url
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False, artistScan=False): if cron and not headphones.CONFIG.LIBRARYSCAN: return if not dir: if not headphones.CONFIG.MUSIC_DIR: return else: dir = headphones.CONFIG.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append or artistScan: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace')) return myDB = db.DBConnection() new_artists = [] logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up bad filepaths tracks = myDB.select( 'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL' ) for track in tracks: encoded_track_string = track['Location'].encode( headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) myDB.action( 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) del_have_tracks = myDB.select( 'SELECT Location, Matched, ArtistName from have') for track in del_have_tracks: encoded_track_string = track['Location'].encode( headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): if track['ArtistName']: # Make sure deleted files get accounted for when updating artist track counts new_artists.append(track['ArtistName']) myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) logger.info( 'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) bitrates = [] song_list = [] latest_subdirectory = [] new_song_count = 0 file_count = 0 for r, d, f in helpers.walk_directory(dir): # Filter paths based on config. Note that these methods work directly # on the inputs helpers.path_filter_patterns(d, headphones.CONFIG.IGNORED_FOLDERS, r) helpers.path_filter_patterns(f, headphones.CONFIG.IGNORED_FILES, r) for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): subdirectory = r.replace(dir, '') latest_subdirectory.append(subdirectory) if file_count == 0 and r.replace(dir, '') != '': logger.info( "[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) elif latest_subdirectory[file_count] != latest_subdirectory[ file_count - 1] and file_count != 0: logger.info( "[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except (FileTypeError, UnreadableFileError): logger.warning( "Cannot read media file '%s', skipping. It may be corrupted or not a media file.", unicode_song_path) continue except IOError: logger.warning( "Cannnot read media file '%s', skipping. Does the file exists?", unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) if f_artist and f.album and f.title: CleanName = helpers.clean_name(f_artist + ' ' + f.album + ' ' + f.title) else: CleanName = None controlValueDict = {'Location': unicode_song_path} newValueDict = { 'TrackID': f.mb_trackid, # 'ReleaseID' : f.mb_albumid, 'ArtistName': f_artist, 'AlbumTitle': f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre': f.genre, 'Date': f.date, 'TrackTitle': f.title, 'BitRate': f.bitrate, 'Format': f.format, 'CleanName': CleanName } # song_list.append(song_dict) check_exist_song = myDB.action( "SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. if not check_exist_song: # This is a new track if f_artist: new_artists.append(f_artist) myDB.upsert("have", newValueDict, controlValueDict) new_song_count += 1 else: if check_exist_song[ 'ArtistName'] != f_artist or check_exist_song[ 'AlbumTitle'] != f.album or check_exist_song[ 'TrackTitle'] != f.title: # Important track metadata has been modified, need to run matcher again if f_artist and f_artist != check_exist_song[ 'ArtistName']: new_artists.append(f_artist) elif f_artist and f_artist == check_exist_song['ArtistName'] and \ check_exist_song['Matched'] != "Ignored": new_artists.append(f_artist) else: continue newValueDict['Matched'] = None myDB.upsert("have", newValueDict, controlValueDict) myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) myDB.action( 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) new_song_count += 1 else: # This track information hasn't changed if f_artist and check_exist_song[ 'Matched'] != "Ignored": new_artists.append(f_artist) file_count += 1 # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) song_list = myDB.action( "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]) total_number_of_songs = \ myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0] logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches # song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 latest_artist = [] last_completion_percentage = 0 for song in song_list: latest_artist.append(song['ArtistName']) if song_count == 0: logger.info("Now matching songs by %s" % song['ArtistName']) elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0: logger.info("Now matching songs by %s" % song['ArtistName']) song_count += 1 completion_percentage = math.floor( float(song_count) / total_number_of_songs * 1000) / 10 if completion_percentage >= (last_completion_percentage + 10): logger.info("Track matching is " + str(completion_percentage) + "% complete") last_completion_percentage = completion_percentage # THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT # EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action( 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle'] ]).fetchone() have_updated = False if track: controlValueDict = { 'ArtistName': track['ArtistName'], 'AlbumTitle': track['AlbumTitle'], 'TrackTitle': track['TrackTitle'] } newValueDict = { 'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: track = myDB.action( 'SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if track: controlValueDict = {'CleanName': track['CleanName']} newValueDict = { 'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True alltrack = myDB.action( 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle'] ]).fetchone() if alltrack: controlValueDict = { 'ArtistName': alltrack['ArtistName'], 'AlbumTitle': alltrack['AlbumTitle'], 'TrackTitle': alltrack['TrackTitle'] } newValueDict = { 'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: alltrack = myDB.action( 'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if alltrack: controlValueDict = {'CleanName': alltrack['CleanName']} newValueDict = { 'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks if not have_updated: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) else: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append or artistScan: logger.info('Updating scanned artist track counts') # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select( 'SELECT ArtistName, ArtistID from artists') # There was a bug where artists with special characters (-,') would show up in new artists. artist_list = [ x for x in unique_artists if helpers.clean_name(x).lower() not in [helpers.clean_name(y[0]).lower() for y in current_artists] ] artists_checked = [ x for x in unique_artists if helpers.clean_name(x).lower() in [helpers.clean_name(y[0]).lower() for y in current_artists] ] # Update track counts for artist in artists_checked: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = (len( myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist]) ) + len( myDB.select( 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist]))) # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases # (can fix by getting rid of second len statement) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) logger.info('Found %i new artists' % len(artist_list)) if artist_list: if headphones.CONFIG.AUTO_ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info( 'To add these artists, go to Manage->Manage New Artists') # myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) if headphones.CONFIG.DETECT_BITRATE and bitrates: headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len( bitrates) / 1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len( myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID]) ) + len( myDB.select( 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) if not append: update_album_status() if not append and not artistScan: lastfm.getSimilar() logger.info('Library scan complete')