def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None): if not dir: dir = headphones.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING)) return myDB = db.DBConnection() if not append: # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) myDB.action('DELETE from have') logger.info('Scanning music directory: %s' % dir) new_artists = [] bitrates = [] song_list = [] for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) song_dict = { 'TrackID' : f.mb_trackid, 'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre' : f.genre, 'Date' : f.date, 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, 'Location' : unicode_song_path } song_list.append(song_dict) # Now we start track matching total_number_of_songs = len(song_list) logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 for song in song_list: song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") # If the track has a trackid & releaseid (beets: albumid) that the most surefire way # of identifying a track to a specific release so we'll use that first if song['TrackID'] and song['ReleaseID']: # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table if not track: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() if track: # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo controlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['ReleaseID'] } # Insert it into the Headphones hybrid release (ReleaseID == AlbumID) hybridControlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['AlbumID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) myDB.upsert("alltracks", newValueDict, hybridControlValueDict) myDB.upsert("tracks", newValueDict, hybridControlValueDict) # Matched. Move on to the next one: continue # If we can't find it with TrackID & ReleaseID, next most specific will be # releaseid + tracktitle, although perhaps less reliable due to a higher # likelihood of variations in the song title (e.g. feat. artists) if song['ReleaseID'] and song['TrackTitle']: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() if not track: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() if track: # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table) # So use both TrackID & ReleaseID as the control values controlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['ReleaseID'] } hybridControlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['AlbumID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } # Update both tables here as well myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) myDB.upsert("alltracks", newValueDict, hybridControlValueDict) myDB.upsert("tracks", newValueDict, hybridControlValueDict) # Done continue # Next most specific will be the opposite: a TrackID and an AlbumTitle # TrackIDs span multiple releases so if something is on an official album # and a compilation, for example, this will match it to the right one # However - there may be multiple matches here if song['TrackID'] and song['AlbumTitle']: # Even though there might be multiple matches, we just need to grab one to confirm a match track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() if not track: track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() if track: # Don't need the hybridControlValueDict here since ReleaseID is not unique controlValueDict = { 'TrackID' : track['TrackID'], 'AlbumTitle' : track['AlbumTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably # even more unreliable than the previous queries, and might span multiple releases) if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if not track: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc) # This is more reliable than the former but requires some string manipulation so we'll do it only # if we can't find a match with the original data if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone() if not track: track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone() if track: controlValueDict = { 'CleanName' : track['CleanName'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track # with more specific information exists in the library, it'll overwrite these values if song['TrackID']: track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone() if not track: track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone() if track: controlValueDict = { 'TrackID' : track['TrackID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release if song['ArtistName']: new_artists.append(song['ArtistName']) else: continue # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) else: continue myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir) if not append: # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info('Updating current artist track counts') for artist in current_artists: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT into newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
def addArtisttoDB(artistid, extrasonly=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() if dbartist is None: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading"} else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} if headphones.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist['releasegroups']: rgid = rg['id'] # check if the album already exists rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rg['id']]) try: release_dict = mb.getReleaseGroup(rgid) except Exception, e: logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue if not release_dict: continue logger.info(u"Now adding/updating album: " + rg['title']) controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['releasedate'], "Type": rg['type'] } # Only change the status & add DateAdded if the album is not already in the database if not len(rg_exists): newValueDict['DateAdded']= helpers.today() if headphones.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif release_dict['releasedate'] > helpers.today() and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) for track in release_dict['tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "AlbumID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) logger.debug(u"Updating album cache for " + rg['title']) cache.getThumb(AlbumID=rg['id'])
def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn("Cannot import blocked special purpose artist with id" + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action("DELETE from blacklist WHERE ArtistID=?", [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action("SELECT * FROM artists WHERE ArtistID=?", [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = { "ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.INCLUDE_EXTRAS, "Extras": headphones.EXTRAS, } else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if artist and artist.get("artist_name") in blacklisted_special_artist_names: logger.warn("Cannot import blocked special purpose artist: %s" % artist.get("artist_name")) myDB.action("DELETE from artists WHERE ArtistID=?", [artistid]) # in case it's already in the db myDB.action("DELETE from albums WHERE ArtistID=?", [artistid]) myDB.action("DELETE from tracks WHERE ArtistID=?", [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist["artist_name"].startswith("The "): sortname = artist["artist_name"][4:] else: sortname = artist["artist_name"] logger.info(u"Now adding/updating: " + artist["artist_name"]) controlValueDict = {"ArtistID": artistid} newValueDict = { "ArtistName": artist["artist_name"], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading", } myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence over global option # Global options are set when adding a new artist myDB = db.DBConnection() try: db_artist = myDB.action("SELECT IncludeExtras, Extras from artists WHERE ArtistID=?", [artistid]).fetchone() includeExtras = db_artist["IncludeExtras"] except IndexError: includeExtras = False # Clean all references to release group in dB that are no longer referenced in musicbrainz group_list = [] force_repackage = 0 # Don't nuke the database if there's a MusicBrainz error if len(artist["releasegroups"]) != 0 and not extrasonly: for groups in artist["releasegroups"]: group_list.append(groups["id"]) remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) for items in remove_missing_groups_from_albums: if items["AlbumID"] not in group_list: # Remove all from albums/tracks that aren't in release groups myDB.action("DELETE FROM albums WHERE AlbumID=?", [items["AlbumID"]]) myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items["AlbumID"]]) myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items["AlbumID"]]) myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items["AlbumID"]]) logger.info( "[%s] Removing all references to release group %s to reflect MusicBrainz" % (artist["artist_name"], items["AlbumID"]) ) force_repackage = 1 else: logger.info("[%s] Error pulling data from MusicBrainz: Maintaining dB" % artist["artist_name"]) # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks for rg in artist["releasegroups"]: al_title = rg["title"] today = helpers.today() rgid = rg["id"] skip_log = 0 # Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.MB_IGNORE_AGE check_release_date = myDB.action( "SELECT ReleaseDate, Status from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title) ).fetchone() # Skip update if Status set if check_release_date and check_release_date[1]: logger.info( "[%s] Not updating: %s (Status is %s, skipping)" % (artist["artist_name"], rg["title"], check_release_date[1]) ) continue if not forcefull: if check_release_date: if check_release_date[0] is None: logger.info("[%s] Now updating: %s (No Release Date)" % (artist["artist_name"], rg["title"])) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: if len(check_release_date[0]) == 10: release_date = check_release_date[0] elif len(check_release_date[0]) == 7: release_date = check_release_date[0] + "-31" elif len(check_release_date[0]) == 4: release_date = check_release_date[0] + "-12-31" else: release_date = today if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta: logger.info( "[%s] Now updating: %s (Release Date <%s Days) " % (artist["artist_name"], rg["title"], pause_delta) ) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: logger.info( "[%s] Skipping: %s (Release Date >%s Days)" % (artist["artist_name"], rg["title"], pause_delta) ) skip_log = 1 new_releases = 0 else: logger.info("[%s] Now adding: %s (New Release Group)" % (artist["artist_name"], rg["title"])) new_releases = mb.get_new_releases(rgid, includeExtras) if force_repackage == 1: new_releases = -1 logger.info("[%s] Forcing repackage of %s (Release Group Removed)" % (artist["artist_name"], al_title)) else: new_releases = new_releases else: logger.info("[%s] Now adding/updating: %s (Comprehensive Force)" % (artist["artist_name"], rg["title"])) new_releases = mb.get_new_releases(rgid, includeExtras, forcefull) # What this does is adds new releases per artist to the allalbums + alltracks databases # new_releases = mb.get_new_releases(rgid,includeExtras) # print al_title # print new_releases if new_releases != 0: # Dump existing hybrid release since we're repackaging/replacing it myDB.action("DELETE from albums WHERE ReleaseID=?", [rg["id"]]) myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg["id"]]) myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg["id"]]) myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg["id"]]) # This will be used later to build a hybrid release fullreleaselist = [] # Search for releases within a release group find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg["id"]]) # Build the dictionary for the fullreleaselist for items in find_hybrid_releases: if ( items["ReleaseID"] != rg["id"] ): # don't include hybrid information, since that's what we're replacing hybrid_release_id = items["ReleaseID"] newValueDict = { "ArtistID": items["ArtistID"], "ArtistName": items["ArtistName"], "AlbumTitle": items["AlbumTitle"], "AlbumID": items["AlbumID"], "AlbumASIN": items["AlbumASIN"], "ReleaseDate": items["ReleaseDate"], "Type": items["Type"], "ReleaseCountry": items["ReleaseCountry"], "ReleaseFormat": items["ReleaseFormat"], } find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] for hybrid_tracks in find_hybrid_tracks: hybrid_track_array.append( { "number": hybrid_tracks["TrackNumber"], "title": hybrid_tracks["TrackTitle"], "id": hybrid_tracks["TrackID"], #'url': hybrid_tracks['TrackURL'], "duration": hybrid_tracks["TrackDuration"], } ) totalTracks += 1 newValueDict["ReleaseID"] = hybrid_release_id newValueDict["Tracks"] = hybrid_track_array fullreleaselist.append(newValueDict) # print fullreleaselist # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) logger.info("[%s] Packaging %s releases into hybrid title" % (artist["artist_name"], rg["title"])) except Exception, e: errors = True logger.warn( "[%s] Unable to get hybrid release information for %s: %s" % (artist["artist_name"], rg["title"], e) ) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg["id"]} newValueDict = { "ArtistID": artistid, "ArtistName": artist["artist_name"], "AlbumTitle": rg["title"], "AlbumID": rg["id"], "AlbumASIN": hybridrelease["AlbumASIN"], "ReleaseDate": hybridrelease["ReleaseDate"], "Type": rg["type"], } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease["Tracks"]: cleanname = helpers.cleanName(artist["artist_name"] + " " + rg["title"] + " " + track["title"]) controlValueDict = {"TrackID": track["id"], "ReleaseID": rg["id"]} newValueDict = { "ArtistID": artistid, "ArtistName": artist["artist_name"], "AlbumTitle": rg["title"], "AlbumASIN": hybridrelease["AlbumASIN"], "AlbumID": rg["id"], "TrackTitle": track["title"], "TrackDuration": track["duration"], "TrackNumber": track["number"], "CleanName": cleanname, } match = myDB.action( "SELECT Location, BitRate, Format from have WHERE CleanName=?", [cleanname] ).fetchone() if not match: match = myDB.action( "SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?", [artist["artist_name"], rg["title"], track["title"]], ).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict["Location"] = match["Location"] newValueDict["BitRate"] = match["BitRate"] newValueDict["Format"] = match["Format"] # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action("UPDATE have SET Matched=? WHERE Location=?", (rg["id"], match["Location"])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table # myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg["id"]]).fetchone() if not rg_exists: releaseid = rg["id"] else: releaseid = rg_exists["ReleaseID"] album = myDB.action("SELECT * from allalbums WHERE ReleaseID=?", [releaseid]).fetchone() controlValueDict = {"AlbumID": rg["id"]} newValueDict = { "ArtistID": album["ArtistID"], "ArtistName": album["ArtistName"], "AlbumTitle": album["AlbumTitle"], "ReleaseID": album["ReleaseID"], "AlbumASIN": album["AlbumASIN"], "ReleaseDate": album["ReleaseDate"], "Type": album["Type"], "ReleaseCountry": album["ReleaseCountry"], "ReleaseFormat": album["ReleaseFormat"], } if not rg_exists: today = helpers.today() newValueDict["DateAdded"] = today if headphones.AUTOWANT_ALL: newValueDict["Status"] = "Wanted" elif album["ReleaseDate"] > today and headphones.AUTOWANT_UPCOMING: newValueDict["Status"] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif ( helpers.get_age(album["ReleaseDate"]) and helpers.get_age(today) - helpers.get_age(album["ReleaseDate"]) < 21 and headphones.AUTOWANT_UPCOMING ): newValueDict["Status"] = "Wanted" else: newValueDict["Status"] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT total_track_count = len(tracks) for track in tracks: controlValueDict = {"TrackID": track["TrackID"], "AlbumID": rg["id"]} newValueDict = { "ArtistID": track["ArtistID"], "ArtistName": track["ArtistName"], "AlbumTitle": track["AlbumTitle"], "AlbumASIN": track["AlbumASIN"], "ReleaseID": track["ReleaseID"], "TrackTitle": track["TrackTitle"], "TrackDuration": track["TrackDuration"], "TrackNumber": track["TrackNumber"], "CleanName": track["CleanName"], "Location": track["Location"], "Format": track["Format"], "BitRate": track["BitRate"], } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len( myDB.select("SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL", [rg["id"]]) ) marked_as_downloaded = False if rg_exists: if rg_exists["Status"] == "Skipped" and ( (have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0) ): myDB.action("UPDATE albums SET Status=? WHERE AlbumID=?", ["Downloaded", rg["id"]]) marked_as_downloaded = True else: if (have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0): myDB.action("UPDATE albums SET Status=? WHERE AlbumID=?", ["Downloaded", rg["id"]]) marked_as_downloaded = True logger.info(u"[%s] Seeing if we need album art for %s" % (artist["artist_name"], rg["title"])) cache.getThumb(AlbumID=rg["id"]) # start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: from headphones import searcher searcher.searchforalbum(albumid=rg["id"]) else: if skip_log == 0: logger.info(u"[%s] No new releases, so no changes made to %s" % (artist["artist_name"], rg["title"]))
"ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "ReleaseDate": release['ReleaseDate'], "Type": release['Type'], "ReleaseCountry": release['ReleaseCountry'], "ReleaseFormat": release['ReleaseFormat'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in release['Tracks']: cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) controlValueDict = { "TrackID": track['id'], "ReleaseID": release['ReleaseID'] } newValueDict = { "ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "TrackTitle": track['title'], "TrackDuration": track['duration'],
def addReleaseById(rid, rgid=None): myDB = db.DBConnection() # Create minimum info upfront if added from searchresults status = '' if rgid: dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) if not dbalbum: status = 'Loading' controlValueDict = {"AlbumID": rgid} newValueDict = {"AlbumTitle": rgid, "ArtistName": status, "Status": status} myDB.upsert("albums", newValueDict, controlValueDict) time.sleep(1) rgid = None artistid = None release_dict = None results = myDB.select("SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid]) for result in results: rgid = result['ReleaseGroupID'] artistid = result['ArtistID'] logger.debug("Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid) if not rgid: #didn't find it in the cache, get the information from MB logger.debug("Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID") try: release_dict = mb.getRelease(rid) except Exception as e: logger.info('Unable to get release information for Release %s: %s', rid, e) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not release_dict: logger.info('Unable to get release information for Release %s: no dict', rid) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return rgid = release_dict['rgid'] artistid = release_dict['artist_id'] #we don't want to make more calls to MB here unless we have to, could be happening quite a lot rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) #make sure the artist exists since I don't know what happens later if it doesn't artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) if not artist_exists and release_dict: if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: sortname = release_dict['artist_name'] logger.info(u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused") controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = {"ArtistName": release_dict['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused"} if headphones.CONFIG.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 newValueDict['Extras'] = headphones.CONFIG.EXTRAS myDB.upsert("artists", newValueDict, controlValueDict) elif not artist_exists and not release_dict: logger.error("Artist does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not rg_exists and release_dict or status == 'Loading' and release_dict: #it should never be the case that we have an rg and not the artist #but if it is this will fail logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid) controlValueDict = {"AlbumID": rgid} if status != 'Loading': status = 'Wanted' newValueDict = {"ArtistID": release_dict['artist_id'], "ReleaseID": rgid, "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'] if 'title' in release_dict else release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['date'], "DateAdded": helpers.today(), "Status": status, "Type": release_dict['rg_type'], "ReleaseID": rid } myDB.upsert("albums", newValueDict, controlValueDict) #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "AlbumID": rgid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() #if not match: #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] #myDB.action('DELETE from have WHERE Location=?', [match['Location']]) # If the album has been scanned before adding the release it will be unmatched, update to matched if match['Matched'] == 'Failed': myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release_dict['rgid'], match['Location'])) myDB.upsert("tracks", newValueDict, controlValueDict) # Reset status if status == 'Loading': controlValueDict = {"AlbumID": rgid} if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: newValueDict = {"Status": "Wanted"} else: newValueDict = {"Status": "Skipped"} myDB.upsert("albums", newValueDict, controlValueDict) # Start a search for the album if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: import searcher searcher.searchforalbum(rgid, False) elif not rg_exists and not release_dict: logger.error("ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return else: logger.info('Release ' + str(rid) + " already exists in the database!")
def addArtisttoDB(artistid, extrasonly=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.INCLUDE_EXTRAS, "Extras": headphones.EXTRAS } else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist['releasegroups']: logger.info("Now adding/updating: " + rg['title']) rgid = rg['id'] # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() try: releaselist = mb.getReleaseGroup(rgid) except Exception, e: logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue if not releaselist: errors = True continue # This will be used later to build a hybrid release fullreleaselist = [] for release in releaselist: # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table releaseid = release['id'] try: releasedict = mb.getRelease(releaseid, include_artist_info=False) except Exception, e: errors = True logger.info('Unable to get release information for %s: %s' % (release['id'], e)) continue if not releasedict: errors = True continue controlValueDict = {"ReleaseID": release['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": releasedict['asin'], "ReleaseDate": releasedict['date'], "Type": rg['type'], "ReleaseCountry": releasedict['country'], "ReleaseFormat": releasedict['format'] } myDB.upsert("allalbums", newValueDict, controlValueDict) # Build the dictionary for the fullreleaselist newValueDict['ReleaseID'] = release['id'] newValueDict['Tracks'] = releasedict['tracks'] fullreleaselist.append(newValueDict) for track in releasedict['tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": release['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": releasedict['asin'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict)
def libraryScan(dir=None): if not dir: dir = headphones.MUSIC_DIR try: dir = str(dir) except UnicodeEncodeError: dir = unicode(dir).encode('unicode_escape') if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir) return myDB = db.DBConnection() # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) logger.info('Scanning music directory: %s' % dir) new_artists = [] bitrates = [] myDB.action('DELETE from have') for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors='replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + file) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Try to find a match based on artist/album/tracktitle if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: continue if f_artist and f.album and f.title: track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone() if not track: track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']]) continue # Try to match on mbid if available and we couldn't find a match based on metadata if f.mb_trackid: # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']]) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release new_artists.append(f_artist) # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist+' '+f.album+' '+f.title), f.format]) logger.info('Completed scanning of directory: %s' % dir) logger.info('Checking filepaths to see if we can find any matches') # Now check empty file paths to see if we can find a match based on their folder format tracks = myDB.select('SELECT * from tracks WHERE Location IS NULL') for track in tracks: release = myDB.action('SELECT * from albums WHERE AlbumID=?', [track['AlbumID']]).fetchone() try: year = release['ReleaseDate'][:4] except TypeError: year = '' artist = release['ArtistName'].replace('/', '_') album = release['AlbumTitle'].replace('/', '_') if release['ArtistName'].startswith('The '): sortname = release['ArtistName'][4:] else: sortname = release['ArtistName'] if sortname.isdigit(): firstchar = '0-9' else: firstchar = sortname[0] lowerfirst = firstchar.lower() albumvalues = { 'artist': artist, 'album': album, 'year': year, 'first': firstchar, 'lowerfirst': lowerfirst } folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues) folder = folder.replace('./', '_/').replace(':','_').replace('?','_') if folder.endswith('.'): folder = folder.replace(folder[len(folder)-1], '_') if not track['TrackNumber']: tracknumber = '' else: tracknumber = '%02d' % track['TrackNumber'] trackvalues = { 'tracknumber': tracknumber, 'title': track['TrackTitle'], 'artist': release['ArtistName'], 'album': release['AlbumTitle'], 'year': year } new_file_name = helpers.replace_all(headphones.FILE_FORMAT, trackvalues).replace('/','_') + '.*' new_file_name = new_file_name.replace('?','_').replace(':', '_') full_path_to_file = os.path.normpath(os.path.join(headphones.MUSIC_DIR, folder, new_file_name)).encode(headphones.SYS_ENCODING, 'replace') match = glob.glob(full_path_to_file) if match: logger.info('Found a match: %s. Writing MBID to metadata' % match[0]) unipath = unicode(match[0], headphones.SYS_ENCODING, errors='replace') myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [unipath, track['TrackID']]) myDB.action('DELETE from have WHERE Location=?', [unipath]) # Try to insert the appropriate track id so we don't have to keep doing this try: f = MediaFile(match[0]) f.mb_trackid = track['TrackID'] f.save() myDB.action('UPDATE tracks SET BitRate=?, Format=? WHERE TrackID=?', [f.bitrate, f.format, track['TrackID']]) logger.debug('Wrote mbid to track: %s' % match[0]) except: logger.error('Error embedding track id into: %s' % match[0]) continue logger.info('Done checking empty filepaths') logger.info('Done syncing library with directory: %s' % dir) # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info('Updating track counts') for artist in current_artists: havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') headphones.NEW_ARTISTS = artist_list if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "ReleaseDate": release['ReleaseDate'], "Type": release['Type'], "ReleaseCountry": release['ReleaseCountry'], "ReleaseFormat": release['ReleaseFormat'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in release['Tracks']: cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": release['ReleaseID']} newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname }
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False): if cron and not headphones.CONFIG.LIBRARYSCAN: return if not dir: if not headphones.CONFIG.MUSIC_DIR: return else: dir = headphones.CONFIG.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace')) return myDB = db.DBConnection() new_artists = [] logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up bad filepaths tracks = myDB.select('SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL') for track in tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) if not os.path.isfile(encoded_track_string): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have') for track in del_have_tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): if track['ArtistName']: # Make sure deleted files get accounted for when updating artist track counts new_artists.append(track['ArtistName']) myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) bitrates = [] song_list = [] latest_subdirectory = [] new_song_count = 0 file_count = 0 for r, d, f in helpers.walk_directory(dir): # Filter paths based on config. Note that these methods work directly # on the inputs helpers.path_filter_patterns(d, headphones.CONFIG.IGNORED_FOLDERS, r) helpers.path_filter_patterns(f, headphones.CONFIG.IGNORED_FILES, r) for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): subdirectory = r.replace(dir, '') latest_subdirectory.append(subdirectory) if file_count == 0 and r.replace(dir, '') != '': logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) elif latest_subdirectory[file_count] != latest_subdirectory[file_count - 1] and file_count != 0: logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except (FileTypeError, UnreadableFileError): logger.warning("Cannot read media file '%s', skipping. It may be corrupted or not a media file.", unicode_song_path) continue except IOError: logger.warning("Cannnot read media file '%s', skipping. Does the file exists?", unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) if f_artist and f.album and f.title: CleanName = helpers.cleanName(f_artist + ' ' + f.album + ' ' + f.title) else: CleanName = None controlValueDict = {'Location': unicode_song_path} newValueDict = {'TrackID': f.mb_trackid, #'ReleaseID' : f.mb_albumid, 'ArtistName': f_artist, 'AlbumTitle': f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre': f.genre, 'Date': f.date, 'TrackTitle': f.title, 'BitRate': f.bitrate, 'Format': f.format, 'CleanName': CleanName } #song_list.append(song_dict) check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. if not check_exist_song: #This is a new track if f_artist: new_artists.append(f_artist) myDB.upsert("have", newValueDict, controlValueDict) new_song_count += 1 else: if check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: #Important track metadata has been modified, need to run matcher again if f_artist and f_artist != check_exist_song['ArtistName']: new_artists.append(f_artist) elif f_artist and f_artist == check_exist_song['ArtistName'] and check_exist_song['Matched'] != "Ignored": new_artists.append(f_artist) else: continue newValueDict['Matched'] = None myDB.upsert("have", newValueDict, controlValueDict) myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) new_song_count += 1 else: #This track information hasn't changed if f_artist and check_exist_song['Matched'] != "Ignored": new_artists.append(f_artist) file_count += 1 # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]) total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0] logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 latest_artist = [] for song in song_list: latest_artist.append(song['ArtistName']) if song_count == 0: logger.info("Now matching songs by %s" % song['ArtistName']) elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0: logger.info("Now matching songs by %s" % song['ArtistName']) song_count += 1 completion_percentage = float(song_count) / total_number_of_songs * 100 if completion_percentage % 10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() have_updated = False if track: controlValueDict = {'ArtistName': track['ArtistName'], 'AlbumTitle': track['AlbumTitle'], 'TrackTitle': track['TrackTitle']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if track: controlValueDict = {'CleanName': track['CleanName']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if alltrack: controlValueDict = {'ArtistName': alltrack['ArtistName'], 'AlbumTitle': alltrack['AlbumTitle'], 'TrackTitle': alltrack['TrackTitle']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if alltrack: controlValueDict = {'CleanName': alltrack['CleanName']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks if not have_updated: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) else: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: logger.info('Updating scanned artist track counts') # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') #There was a bug where artists with special characters (-,') would show up in new artists. artist_list = [ x for x in unique_artists if helpers.cleanName(x).lower() not in [ helpers.cleanName(y[0]).lower() for y in current_artists ] ] artists_checked = [ x for x in unique_artists if helpers.cleanName(x).lower() in [ helpers.cleanName(y[0]).lower() for y in current_artists ] ] # Update track counts for artist in artists_checked: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = ( len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist])) ) # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases # (can fix by getting rid of second len statement) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) logger.info('Found %i new artists' % len(artist_list)) if artist_list: if headphones.CONFIG.AUTO_ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') #myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) if headphones.CONFIG.DETECT_BITRATE: headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) if not append: update_album_status() lastfm.getSimilar() logger.info('Library scan complete')
def libraryScan(dir=None): if not dir: dir = headphones.MUSIC_DIR dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING)) return myDB = db.DBConnection() # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) logger.info('Scanning music directory: %s' % dir) new_artists = [] bitrates = [] myDB.action('DELETE from have') for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Try to find a match based on artist/album/tracktitle if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: continue if f_artist and f.album and f.title: track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone() if not track: track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']]) continue # Try to match on mbid if available and we couldn't find a match based on metadata if f.mb_trackid: # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']]) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release new_artists.append(f_artist) # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, unicode_song_path, helpers.cleanName(f_artist+' '+f.album+' '+f.title), f.format]) logger.info('Completed scanning directory: %s' % dir) # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info('Updating track counts') for artist in current_artists: havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT into newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None): if not dir: dir = headphones.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace')) return myDB = db.DBConnection() if not append: # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) myDB.action('DELETE from have') logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) new_artists = [] bitrates = [] song_list = [] for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) song_dict = { 'TrackID' : f.mb_trackid, 'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre' : f.genre, 'Date' : f.date, 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, 'Location' : unicode_song_path } song_list.append(song_dict) # Now we start track matching total_number_of_songs = len(song_list) logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 for song in song_list: song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") # If the track has a trackid & releaseid (beets: albumid) that the most surefire way # of identifying a track to a specific release so we'll use that first if song['TrackID'] and song['ReleaseID']: # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table if not track: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() if track: # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo controlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['ReleaseID'] } # Insert it into the Headphones hybrid release (ReleaseID == AlbumID) hybridControlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['AlbumID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) myDB.upsert("alltracks", newValueDict, hybridControlValueDict) myDB.upsert("tracks", newValueDict, hybridControlValueDict) # Matched. Move on to the next one: continue # If we can't find it with TrackID & ReleaseID, next most specific will be # releaseid + tracktitle, although perhaps less reliable due to a higher # likelihood of variations in the song title (e.g. feat. artists) if song['ReleaseID'] and song['TrackTitle']: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() if not track: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() if track: # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table) # So use both TrackID & ReleaseID as the control values controlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['ReleaseID'] } hybridControlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['AlbumID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } # Update both tables here as well myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) myDB.upsert("alltracks", newValueDict, hybridControlValueDict) myDB.upsert("tracks", newValueDict, hybridControlValueDict) # Done continue # Next most specific will be the opposite: a TrackID and an AlbumTitle # TrackIDs span multiple releases so if something is on an official album # and a compilation, for example, this will match it to the right one # However - there may be multiple matches here if song['TrackID'] and song['AlbumTitle']: # Even though there might be multiple matches, we just need to grab one to confirm a match track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() if not track: track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() if track: # Don't need the hybridControlValueDict here since ReleaseID is not unique controlValueDict = { 'TrackID' : track['TrackID'], 'AlbumTitle' : track['AlbumTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably # even more unreliable than the previous queries, and might span multiple releases) if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if not track: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc) # This is more reliable than the former but requires some string manipulation so we'll do it only # if we can't find a match with the original data if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone() if not track: track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone() if track: controlValueDict = { 'CleanName' : track['CleanName'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track # with more specific information exists in the library, it'll overwrite these values if song['TrackID']: track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone() if not track: track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone() if track: controlValueDict = { 'TrackID' : track['TrackID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release if song['ArtistName']: new_artists.append(song['ArtistName']) else: continue # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) else: continue myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info('Updating current artist track counts') for artist in current_artists: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT into newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
def scrapeScan(self, dir=None): for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors='replace') # Try to read the metadata try: f = MediaFile(song) except Exception, e: logger.error('Got Exception: ' + str(e)) logger.error('Cannot read file: ' + file) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Try to find a match based on artist/album/tracktitle if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: continue if f_artist and f.album and f.title: track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone() if not track: track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) continue # Try to match on mbid if available and we couldn't find a match based on metadata if f.mb_trackid: # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release new_artists.append(f_artist) # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database myDB.action('INSERT INTO have VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist+' '+f.album+' '+f.title)])
def addArtisttoDB(artistid, extrasonly=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = { "ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.INCLUDE_EXTRAS, "Extras": headphones.EXTRAS } else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = { "ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active" } else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = { "ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading" } myDB.upsert("artists", newValueDict, controlValueDict) for rg in artist['releasegroups']: logger.info("Now adding/updating: " + rg['title']) rgid = rg['id'] # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() try: releaselist = mb.getReleaseGroup(rgid) except Exception, e: logger.info( 'Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue if not releaselist: errors = True continue # This will be used later to build a hybrid release fullreleaselist = [] for release in releaselist: # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table releaseid = release['id'] try: releasedict = mb.getRelease(releaseid, include_artist_info=False) except Exception, e: errors = True logger.info('Unable to get release information for %s: %s' % (release['id'], e)) continue if not releasedict: errors = True continue controlValueDict = {"ReleaseID": release['id']} newValueDict = { "ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": releasedict['asin'], "ReleaseDate": releasedict['date'], "Type": rg['type'], "ReleaseCountry": releasedict['country'], "ReleaseFormat": releasedict['format'] } myDB.upsert("allalbums", newValueDict, controlValueDict) # Build the dictionary for the fullreleaselist newValueDict['ReleaseID'] = release['id'] newValueDict['Tracks'] = releasedict['tracks'] fullreleaselist.append(newValueDict) for track in releasedict['tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = { "TrackID": track['id'], "ReleaseID": release['id'] } newValueDict = { "ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": releasedict['asin'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title'] ]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action( 'UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict)
"AlbumTitle": release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['date'], "DateAdded": helpers.today(), "Status": 'Wanted', "Type": release_dict['rg_type'] } myDB.upsert("albums", newValueDict, controlValueDict) #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "AlbumID": rgid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
def addReleaseById(rid, rgid=None): myDB = db.DBConnection() # Create minimum info upfront if added from searchresults status = '' if rgid: dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) if not dbalbum: status = 'Loading' controlValueDict = {"AlbumID": rgid} newValueDict = {"AlbumTitle": rgid, "ArtistName": status, "Status": status} myDB.upsert("albums", newValueDict, controlValueDict) time.sleep(1) rgid = None artistid = None release_dict = None results = myDB.select("SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid]) for result in results: rgid = result['ReleaseGroupID'] artistid = result['ArtistID'] logger.debug("Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid) if not rgid: #didn't find it in the cache, get the information from MB logger.debug("Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID") try: release_dict = mb.getRelease(rid) except Exception as e: logger.info('Unable to get release information for Release %s: %s', rid, e) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not release_dict: logger.info('Unable to get release information for Release %s: no dict', rid) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return rgid = release_dict['rgid'] artistid = release_dict['artist_id'] #we don't want to make more calls to MB here unless we have to, could be happening quite a lot rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) #make sure the artist exists since I don't know what happens later if it doesn't artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) if not artist_exists and release_dict: if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: sortname = release_dict['artist_name'] logger.info(u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused") controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = {"ArtistName": release_dict['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused"} if headphones.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 newValueDict['Extras'] = headphones.EXTRAS myDB.upsert("artists", newValueDict, controlValueDict) elif not artist_exists and not release_dict: logger.error("Artist does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not rg_exists and release_dict or status == 'Loading' and release_dict: #it should never be the case that we have an rg and not the artist #but if it is this will fail logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid) controlValueDict = {"AlbumID": rgid} if status != 'Loading': status = 'Wanted' newValueDict = {"ArtistID": release_dict['artist_id'], "ReleaseID": rgid, "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['date'], "DateAdded": helpers.today(), "Status": status, "Type": release_dict['rg_type'], "ReleaseID": rid } myDB.upsert("albums", newValueDict, controlValueDict) #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "AlbumID": rgid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() #if not match: #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] #myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) # Reset status if status == 'Loading': controlValueDict = {"AlbumID": rgid} newValueDict = {"Status": "Wanted"} myDB.upsert("albums", newValueDict, controlValueDict) # Start a search for the album import searcher searcher.searchforalbum(rgid, False) elif not rg_exists and not release_dict: logger.error("ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return else: logger.info('Release ' + str(rid) + " already exists in the database!")
def addArtisttoDB(artistid, extrasonly=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn('Cannot import blocked special purpose artist with id' + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.INCLUDE_EXTRAS, "Extras": headphones.EXTRAS } else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if artist and artist.get('artist_name') in blacklisted_special_artist_names: logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name')) myDB.action('DELETE from artists WHERE ArtistID=?', [artistid]) #in case it's already in the db myDB.action('DELETE from albums WHERE ArtistID=?', [artistid]) myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence over global option # Global options are set when adding a new artist myDB = db.DBConnection() try: db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False for rg in artist['releasegroups']: logger.info("Now adding/updating: " + rg['title']) rgid = rg['id'] # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() releases = mb.get_all_releases(rgid,includeExtras) if releases == []: logger.info('No official releases in release group %s' % rg['title']) continue if not releases: errors = True logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue # This will be used later to build a hybrid release fullreleaselist = [] for release in releases: # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table controlValueDict = {"ReleaseID" : release['ReleaseID']} newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "ReleaseDate": release['ReleaseDate'], "Type": release['Type'], "ReleaseCountry": release['ReleaseCountry'], "ReleaseFormat": release['ReleaseFormat'] } myDB.upsert("allalbums", newValueDict, controlValueDict) # Build the dictionary for the fullreleaselist newValueDict['ReleaseID'] = release['ReleaseID'] newValueDict['Tracks'] = release['Tracks'] fullreleaselist.append(newValueDict) for track in release['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": release['ReleaseID']} newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) except Exception, e: errors = True logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) if not rg_exists: releaseid = rg['id'] elif rg_exists and not rg_exists['ReleaseID']: # Need to do some importing here - to transition the old format of using the release group # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID # so we'll need to move over the locations, bitrates & formats from the tracks table to the new # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups logger.info("Copying current track information to alternate releases") tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() for track in tracks: if track['Location']: controlValueDict = {"TrackID": track['TrackID']} newValueDict = {"Location": track['Location'], "BitRate": track['BitRate'], "Format": track['Format'], } myDB.upsert("alltracks", newValueDict, controlValueDict) releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if not rg_exists: today = helpers.today() newValueDict['DateAdded']= today if headphones.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT total_track_count = len(tracks) for track in tracks: controlValueDict = {"TrackID": track['TrackID'], "AlbumID": rg['id']} newValueDict = {"ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) marked_as_downloaded = False if rg_exists: if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True else: if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True logger.info(u"Seeing if we need album art for " + rg['title']) cache.getThumb(AlbumID=rg['id']) #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: from headphones import searcher searcher.searchforalbum(albumid=rg['id'])
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False, artistScan=False): if cron and not headphones.CONFIG.LIBRARYSCAN: return if not dir: if not headphones.CONFIG.MUSIC_DIR: return else: dir = headphones.CONFIG.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append or artistScan: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace')) return myDB = db.DBConnection() new_artists = [] logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up bad filepaths tracks = myDB.select( 'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL' ) for track in tracks: encoded_track_string = track['Location'].encode( headphones.SYS_ENCODING) if not os.path.isfile(encoded_track_string): myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) myDB.action( 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) del_have_tracks = myDB.select( 'SELECT Location, Matched, ArtistName from have') for track in del_have_tracks: encoded_track_string = track['Location'].encode( headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): if track['ArtistName']: # Make sure deleted files get accounted for when updating artist track counts new_artists.append(track['ArtistName']) myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) logger.info( 'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) bitrates = [] song_list = [] latest_subdirectory = [] new_song_count = 0 file_count = 0 for r, d, f in helpers.walk_directory(dir): # Filter paths based on config. Note that these methods work directly # on the inputs helpers.path_filter_patterns(d, headphones.CONFIG.IGNORED_FOLDERS, r) helpers.path_filter_patterns(f, headphones.CONFIG.IGNORED_FILES, r) for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): subdirectory = r.replace(dir, '') latest_subdirectory.append(subdirectory) if file_count == 0 and r.replace(dir, '') != '': logger.info( "[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) elif latest_subdirectory[file_count] != latest_subdirectory[ file_count - 1] and file_count != 0: logger.info( "[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except (FileTypeError, UnreadableFileError): logger.warning( "Cannot read media file '%s', skipping. It may be corrupted or not a media file.", unicode_song_path) continue except IOError: logger.warning( "Cannnot read media file '%s', skipping. Does the file exists?", unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) if f_artist and f.album and f.title: CleanName = helpers.cleanName(f_artist + ' ' + f.album + ' ' + f.title) else: CleanName = None controlValueDict = {'Location': unicode_song_path} newValueDict = { 'TrackID': f.mb_trackid, #'ReleaseID' : f.mb_albumid, 'ArtistName': f_artist, 'AlbumTitle': f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre': f.genre, 'Date': f.date, 'TrackTitle': f.title, 'BitRate': f.bitrate, 'Format': f.format, 'CleanName': CleanName } #song_list.append(song_dict) check_exist_song = myDB.action( "SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. if not check_exist_song: #This is a new track if f_artist: new_artists.append(f_artist) myDB.upsert("have", newValueDict, controlValueDict) new_song_count += 1 else: if check_exist_song[ 'ArtistName'] != f_artist or check_exist_song[ 'AlbumTitle'] != f.album or check_exist_song[ 'TrackTitle'] != f.title: #Important track metadata has been modified, need to run matcher again if f_artist and f_artist != check_exist_song[ 'ArtistName']: new_artists.append(f_artist) elif f_artist and f_artist == check_exist_song[ 'ArtistName'] and check_exist_song[ 'Matched'] != "Ignored": new_artists.append(f_artist) else: continue newValueDict['Matched'] = None myDB.upsert("have", newValueDict, controlValueDict) myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) myDB.action( 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) new_song_count += 1 else: #This track information hasn't changed if f_artist and check_exist_song[ 'Matched'] != "Ignored": new_artists.append(f_artist) file_count += 1 # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) song_list = myDB.action( "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]) total_number_of_songs = myDB.action( "SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0] logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 latest_artist = [] for song in song_list: latest_artist.append(song['ArtistName']) if song_count == 0: logger.info("Now matching songs by %s" % song['ArtistName']) elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0: logger.info("Now matching songs by %s" % song['ArtistName']) song_count += 1 completion_percentage = float(song_count) / total_number_of_songs * 100 if completion_percentage % 10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action( 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle'] ]).fetchone() have_updated = False if track: controlValueDict = { 'ArtistName': track['ArtistName'], 'AlbumTitle': track['AlbumTitle'], 'TrackTitle': track['TrackTitle'] } newValueDict = { 'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: track = myDB.action( 'SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if track: controlValueDict = {'CleanName': track['CleanName']} newValueDict = { 'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True alltrack = myDB.action( 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle'] ]).fetchone() if alltrack: controlValueDict = { 'ArtistName': alltrack['ArtistName'], 'AlbumTitle': alltrack['AlbumTitle'], 'TrackTitle': alltrack['TrackTitle'] } newValueDict = { 'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: alltrack = myDB.action( 'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if alltrack: controlValueDict = {'CleanName': alltrack['CleanName']} newValueDict = { 'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks if not have_updated: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) else: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append or artistScan: logger.info('Updating scanned artist track counts') # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select( 'SELECT ArtistName, ArtistID from artists') #There was a bug where artists with special characters (-,') would show up in new artists. artist_list = [ x for x in unique_artists if helpers.cleanName(x).lower() not in [helpers.cleanName(y[0]).lower() for y in current_artists] ] artists_checked = [ x for x in unique_artists if helpers.cleanName(x).lower() in [helpers.cleanName(y[0]).lower() for y in current_artists] ] # Update track counts for artist in artists_checked: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = (len( myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist]) ) + len( myDB.select( 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist]))) # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases # (can fix by getting rid of second len statement) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) logger.info('Found %i new artists' % len(artist_list)) if artist_list: if headphones.CONFIG.AUTO_ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info( 'To add these artists, go to Manage->Manage New Artists') #myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) if headphones.CONFIG.DETECT_BITRATE and bitrates: headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len( bitrates) / 1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len( myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID]) ) + len( myDB.select( 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) if not append: update_album_status() if not append and not artistScan: lastfm.getSimilar() logger.info('Library scan complete')
def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn('Cannot import blocked special purpose artist with id' + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS, "Extras": headphones.CONFIG.EXTRAS} else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if artist and artist.get('artist_name') in blacklisted_special_artist_names: logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name')) myDB.action('DELETE from artists WHERE ArtistID=?', [artistid]) #in case it's already in the db myDB.action('DELETE from albums WHERE ArtistID=?', [artistid]) myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence # over global option. Global options are set when adding a new artist try: db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False # Clean all references to release group in dB that are no longer referenced # from the musicbrainz refresh group_list = [] force_repackage = 0 # Don't nuke the database if there's a MusicBrainz error if len(artist['releasegroups']) != 0: for groups in artist['releasegroups']: group_list.append(groups['id']) if not extrasonly: remove_missing_groups_from_albums = myDB.select("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) else: remove_missing_groups_from_albums = myDB.select('SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid]) for items in remove_missing_groups_from_albums: if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']]) logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (artist['artist_name'], items['AlbumID'])) if not extrasonly: force_repackage = 1 else: if not extrasonly: logger.info("[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name']) # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks album_searches = [] for rg in artist['releasegroups']: al_title = rg['title'] today = helpers.today() rgid = rg['id'] skip_log = 0 #Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.CONFIG.MB_IGNORE_AGE rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() if not forcefull: new_release_group = False try: check_release_date = rg_exists['ReleaseDate'] except TypeError: check_release_date = None new_release_group = True if new_release_group: logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid, includeExtras) else: if check_release_date is None or check_release_date == u"None": logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: if len(check_release_date) == 10: release_date = check_release_date elif len(check_release_date) == 7: release_date = check_release_date + "-31" elif len(check_release_date) == 4: release_date = check_release_date + "-12-31" else: release_date = today if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta: logger.info("[%s] Now updating: %s (Release Date <%s Days)", artist['artist_name'], rg['title'], pause_delta) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: logger.info("[%s] Skipping: %s (Release Date >%s Days)", artist['artist_name'], rg['title'], pause_delta) skip_log = 1 new_releases = 0 if force_repackage == 1: new_releases = -1 logger.info('[%s] Forcing repackage of %s (Release Group Removed)', artist['artist_name'], al_title) else: new_releases = new_releases else: logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'], rg['title']) new_releases = mb.get_new_releases(rgid, includeExtras, forcefull) if new_releases != 0: # Dump existing hybrid release since we're repackaging/replacing it myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rg['id']]) # This will be used later to build a hybrid release fullreleaselist = [] # Search for releases within a release group find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) # Build the dictionary for the fullreleaselist for items in find_hybrid_releases: if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing hybrid_release_id = items['ReleaseID'] newValueDict = {"ArtistID": items['ArtistID'], "ArtistName": items['ArtistName'], "AlbumTitle": items['AlbumTitle'], "AlbumID": items['AlbumID'], "AlbumASIN": items['AlbumASIN'], "ReleaseDate": items['ReleaseDate'], "Type": items['Type'], "ReleaseCountry": items['ReleaseCountry'], "ReleaseFormat": items['ReleaseFormat'] } find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] for hybrid_tracks in find_hybrid_tracks: hybrid_track_array.append({ 'number': hybrid_tracks['TrackNumber'], 'title': hybrid_tracks['TrackTitle'], 'id': hybrid_tracks['TrackID'], #'url': hybrid_tracks['TrackURL'], 'duration': hybrid_tracks['TrackDuration'] }) totalTracks += 1 newValueDict['ReleaseID'] = hybrid_release_id newValueDict['Tracks'] = hybrid_track_array fullreleaselist.append(newValueDict) # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title'])) except Exception as e: errors = True logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'], rg['title'], e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() #if not match: #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table #myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) # check if the album already exists if not rg_exists: releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] if not releaseid: releaseid = rg['id'] album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if rg_exists: newValueDict['DateAdded'] = rg_exists['DateAdded'] newValueDict['Status'] = rg_exists['Status'] else: today = helpers.today() newValueDict['DateAdded'] = today if headphones.CONFIG.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to # mark it as downloaded. Default is 80%, can be set in config as # ALBUM_COMPLETION_PCT total_track_count = len(tracks) if total_track_count == 0: logger.warning("Total track count is zero for Release ID " + "'%s', skipping.", releaseid) continue for track in tracks: controlValueDict = {"TrackID": track['TrackID'], "AlbumID": rg['id']} newValueDict = {"ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) marked_as_downloaded = False if rg_exists: if rg_exists['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True else: if ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title'])) cache.getThumb(AlbumID=rg['id']) # Start a search for the album if it's new, hasn't been marked as # downloaded and autowant_all is selected. This search is deferred, # in case the search failes and the rest of the import will halt. if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL: album_searches.append(rg['id']) else: if skip_log == 0: logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title'])) time.sleep(3) finalize_update(artistid, artist['artist_name'], errors) logger.info(u"Seeing if we need album art for: %s" % artist['artist_name']) cache.getThumb(ArtistID=artistid) if errors: logger.info("[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name'])) else: myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']]) logger.info(u"Updating complete for: %s" % artist['artist_name']) # Start searching for newly added albums if album_searches: from headphones import searcher logger.info("Start searching for %d albums.", len(album_searches)) for album_search in album_searches: searcher.searchforalbum(albumid=album_search)
def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn('Cannot import blocked special purpose artist with id' + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.INCLUDE_EXTRAS, "Extras": headphones.EXTRAS } else: newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) artist = mb.getArtist(artistid, extrasonly) if artist and artist.get('artist_name') in blacklisted_special_artist_names: logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name')) myDB.action('DELETE from artists WHERE ArtistID=?', [artistid]) #in case it's already in the db myDB.action('DELETE from albums WHERE ArtistID=?', [artistid]) myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence over global option # Global options are set when adding a new artist myDB = db.DBConnection() try: db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False #Clean all references to release group in dB that are no longer referenced in musicbrainz group_list = [] force_repackage = 0 #Don't nuke the database if there's a MusicBrainz error if len(artist['releasegroups']) != 0 and not extrasonly: for groups in artist['releasegroups']: group_list.append(groups['id']) remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) for items in remove_missing_groups_from_albums: if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz" % (artist['artist_name'], items['AlbumID'])) force_repackage = 1 else: logger.info("[%s] Error pulling data from MusicBrainz: Maintaining dB" % artist['artist_name']) # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks for rg in artist['releasegroups']: al_title = rg['title'] today = helpers.today() rgid = rg['id'] skip_log = 0 #Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.MB_IGNORE_AGE if not forcefull: check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone() if check_release_date: if check_release_date[0] is None: logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid,includeExtras,True) else: if len(check_release_date[0]) == 10: release_date = check_release_date[0] elif len(check_release_date[0]) == 7: release_date = check_release_date[0]+"-31" elif len(check_release_date[0]) == 4: release_date = check_release_date[0]+"-12-31" else: release_date = today if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta: logger.info("[%s] Now updating: %s (Release Date <%s Days) " % (artist['artist_name'], rg['title'], pause_delta)) new_releases = mb.get_new_releases(rgid,includeExtras,True) else: logger.info("[%s] Skipping: %s (Release Date >%s Days)" % (artist['artist_name'], rg['title'], pause_delta)) skip_log = 1 new_releases = 0 else: logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid,includeExtras) if force_repackage == 1: new_releases = -1 logger.info('[%s] Forcing repackage of %s (Release Group Removed)' % (artist['artist_name'], al_title)) else: new_releases = new_releases else: logger.info("[%s] Now adding/updating: %s (Comprehensive Force)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid,includeExtras,forcefull) #What this does is adds new releases per artist to the allalbums + alltracks databases #new_releases = mb.get_new_releases(rgid,includeExtras) #print al_title #print new_releases if new_releases != 0: #Dump existing hybrid release since we're repackaging/replacing it myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) # This will be used later to build a hybrid release fullreleaselist = [] #Search for releases within a release group find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) # Build the dictionary for the fullreleaselist for items in find_hybrid_releases: if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing hybrid_release_id = items['ReleaseID'] newValueDict = {"ArtistID": items['ArtistID'], "ArtistName": items['ArtistName'], "AlbumTitle": items['AlbumTitle'], "AlbumID": items['AlbumID'], "AlbumASIN": items['AlbumASIN'], "ReleaseDate": items['ReleaseDate'], "Type": items['Type'], "ReleaseCountry": items['ReleaseCountry'], "ReleaseFormat": items['ReleaseFormat'] } find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] for hybrid_tracks in find_hybrid_tracks: hybrid_track_array.append({ 'number': hybrid_tracks['TrackNumber'], 'title': hybrid_tracks['TrackTitle'], 'id': hybrid_tracks['TrackID'], #'url': hybrid_tracks['TrackURL'], 'duration': hybrid_tracks['TrackDuration'] }) totalTracks += 1 newValueDict['ReleaseID'] = hybrid_release_id newValueDict['Tracks'] = hybrid_track_array fullreleaselist.append(newValueDict) #print fullreleaselist # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title'])) except Exception, e: errors = True logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'],rg['title'],e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() #if not match: #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table #myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() if not rg_exists: releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if not rg_exists: today = helpers.today() newValueDict['DateAdded']= today if headphones.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT total_track_count = len(tracks) for track in tracks: controlValueDict = {"TrackID": track['TrackID'], "AlbumID": rg['id']} newValueDict = {"ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) marked_as_downloaded = False if rg_exists: if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True else: if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title'])) cache.getThumb(AlbumID=rg['id']) #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: from headphones import searcher searcher.searchforalbum(albumid=rg['id']) else: if skip_log == 0: logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title']))
controlValueDict = {"ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname }
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False): if cron and not headphones.LIBRARYSCAN: return if not dir: if not headphones.MUSIC_DIR: return else: dir = headphones.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace')) return myDB = db.DBConnection() if not append: # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL') for track in tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) if not os.path.isfile(encoded_track_string): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) del_have_tracks = myDB.select('SELECT Location, Matched from have') for track in del_have_tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) if not os.path.isfile(encoded_track_string): myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) myDB.action('UPDATE have SET Matched=NULL WHERE Matched=?', [track['Matched']]) logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) ###############myDB.action('DELETE from have') logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) new_artists = [] bitrates = [] song_list = [] new_song_count = 0 for r,d,f in os.walk(dir): #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one #using a list comprehension will not work correctly for nested subdirectories (os.walk keeps its original list) for directory in d[:]: if directory.startswith("."): d.remove(directory) for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) if f_artist and f.album and f.title: CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title) else: CleanName = None controlValueDict = {'Location' : unicode_song_path} newValueDict = { 'TrackID' : f.mb_trackid, #'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre' : f.genre, 'Date' : f.date, 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, 'CleanName' : CleanName } #song_list.append(song_dict) check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. if not check_exist_song: myDB.upsert("have", newValueDict, controlValueDict) new_song_count+=1 elif check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: newValueDict['Matched'] = None myDB.upsert("have", newValueDict, controlValueDict) new_song_count+=1 # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]) total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0] logger.info("Found " + str(total_number_of_songs) + " unmatched tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 latest_artist = [] for song in song_list: latest_artist.append(song['ArtistName']) if song_count == 0: logger.info("Now matching songs by %s" % song['ArtistName']) elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0: logger.info("Now matching songs by %s" % song['ArtistName']) #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle'] song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = { 'ArtistName' : song['ArtistName'], 'AlbumTitle' : song['AlbumTitle'], 'TrackTitle' : song['TrackTitle'] } newValueDict2 = { 'Matched' : track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if track: controlValueDict = { 'CleanName' : track['CleanName']} newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = { 'CleanName' : song['CleanName']} newValueDict2 = { 'Matched' : track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if alltrack: controlValueDict = { 'ArtistName' : alltrack['ArtistName'], 'AlbumTitle' : alltrack['AlbumTitle'], 'TrackTitle' : alltrack['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = { 'ArtistName' : song['ArtistName'], 'AlbumTitle' : song['AlbumTitle'], 'TrackTitle' : song['TrackTitle'] } newValueDict2 = { 'Matched' : alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if alltrack: controlValueDict = { 'CleanName' : alltrack['CleanName']} newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = { 'CleanName' : song['CleanName']} newValueDict2 = { 'Matched' : alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release if song['ArtistName']: new_artists.append(song['ArtistName']) else: continue #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') #There was a bug where artists with special characters (-,') would show up in new artists. artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]] # Update track counts logger.info('Updating current artist track counts') for artist in current_artists: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT into newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) update_album_status() logger.info('Library scan complete')
"AlbumASIN": release_dict["asin"], "ReleaseDate": release_dict["date"], "DateAdded": helpers.today(), "Status": "Wanted", "Type": release_dict["rg_type"], } myDB.upsert("albums", newValueDict, controlValueDict) # keep a local cache of these so that external programs that are adding releasesByID don't hammer MB myDB.action("INSERT INTO releases VALUES( ?, ?)", [rid, release_dict["rgid"]]) for track in release_dict["tracks"]: cleanname = helpers.cleanName( release_dict["artist_name"] + " " + release_dict["rg_title"] + " " + track["title"] ) controlValueDict = {"TrackID": track["id"], "AlbumID": rgid} newValueDict = { "ArtistID": release_dict["artist_id"], "ArtistName": release_dict["artist_name"], "AlbumTitle": release_dict["rg_title"], "AlbumASIN": release_dict["asin"], "TrackTitle": track["title"], "TrackDuration": track["duration"], "TrackNumber": track["number"], "CleanName": cleanname, } match = myDB.action("SELECT Location, BitRate, Format from have WHERE CleanName=?", [cleanname]).fetchone()
def libraryScan(dir=None): if not dir: dir = headphones.MUSIC_DIR try: dir = str(dir) except UnicodeEncodeError: dir = unicode(dir).encode("unicode_escape") if not os.path.isdir(dir): logger.warn("Cannot find directory: %s. Not scanning" % dir) return myDB = db.DBConnection() # Clean up bad filepaths tracks = myDB.select("SELECT Location, TrackID from tracks WHERE Location IS NOT NULL") for track in tracks: if not os.path.isfile(track["Location"].encode(headphones.SYS_ENCODING)): myDB.action( "UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?", [None, None, None, track["TrackID"]], ) logger.info("Scanning music directory: %s" % dir) new_artists = [] bitrates = [] myDB.action("DELETE from have") for r, d, f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith("." + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors="replace") # Try to read the metadata try: f = MediaFile(song) except: logger.error("Cannot read file: " + file) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Try to find a match based on artist/album/tracktitle if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: continue if f_artist and f.album and f.title: track = myDB.action( "SELECT TrackID from tracks WHERE CleanName LIKE ?", [helpers.cleanName(f_artist + " " + f.album + " " + f.title)], ).fetchone() if not track: track = myDB.action( "SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?", [f_artist, f.album, f.title], ).fetchone() if track: myDB.action( "UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?", [file, f.bitrate, f.format, track["TrackID"]], ) continue # Try to match on mbid if available and we couldn't find a match based on metadata if f.mb_trackid: # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action("SELECT TrackID from tracks WHERE TrackID=?", [f.mb_trackid]).fetchone() if track: myDB.action( "UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?", [file, f.bitrate, f.format, track["TrackID"]], ) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release new_artists.append(f_artist) # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database myDB.action( "INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [ f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist + " " + f.album + " " + f.title), f.format, ], ) logger.info("Completed scanning of directory: %s" % dir) logger.info("Checking filepaths to see if we can find any matches") # Now check empty file paths to see if we can find a match based on their folder format tracks = myDB.select("SELECT * from tracks WHERE Location IS NULL") for track in tracks: release = myDB.action("SELECT * from albums WHERE AlbumID=?", [track["AlbumID"]]).fetchone() try: year = release["ReleaseDate"][:4] except TypeError: year = "" artist = release["ArtistName"].replace("/", "_") album = release["AlbumTitle"].replace("/", "_") releasetype = release["Type"].replace("/", "_") if release["ArtistName"].startswith("The "): sortname = release["ArtistName"][4:] else: sortname = release["ArtistName"] if sortname.isdigit(): firstchar = "0-9" else: firstchar = sortname[0] albumvalues = { "$Artist": artist, "$Album": album, "$Year": year, "$Type": releasetype, "$First": firstchar, "$artist": artist.lower(), "$album": album.lower(), "$year": year, "$type": releasetype.lower(), "$first": firstchar.lower(), } folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues) folder = folder.replace("./", "_/").replace(":", "_").replace("?", "_") if folder.endswith("."): folder = folder.replace(folder[len(folder) - 1], "_") if not track["TrackNumber"]: tracknumber = "" else: tracknumber = "%02d" % track["TrackNumber"] title = track["TrackTitle"] trackvalues = { "$Track": tracknumber, "$Title": title, "$Artist": release["ArtistName"], "$Album": release["AlbumTitle"], "$Year": year, "$track": tracknumber, "$title": title.lower(), "$artist": release["ArtistName"].lower(), "$album": release["AlbumTitle"].lower(), "$year": year, } new_file_name = helpers.replace_all(headphones.FILE_FORMAT, trackvalues).replace("/", "_") + ".*" new_file_name = new_file_name.replace("?", "_").replace(":", "_") full_path_to_file = os.path.normpath(os.path.join(headphones.MUSIC_DIR, folder, new_file_name)).encode( headphones.SYS_ENCODING, "replace" ) match = glob.glob(full_path_to_file) if match: logger.info("Found a match: %s. Writing MBID to metadata" % match[0]) unipath = unicode(match[0], headphones.SYS_ENCODING, errors="replace") myDB.action("UPDATE tracks SET Location=? WHERE TrackID=?", [unipath, track["TrackID"]]) myDB.action("DELETE from have WHERE Location=?", [unipath]) # Try to insert the appropriate track id so we don't have to keep doing this try: f = MediaFile(match[0]) f.mb_trackid = track["TrackID"] f.save() myDB.action( "UPDATE tracks SET BitRate=?, Format=? WHERE TrackID=?", [f.bitrate, f.format, track["TrackID"]] ) logger.debug("Wrote mbid to track: %s" % match[0]) except: logger.error("Error embedding track id into: %s" % match[0]) continue logger.info("Done checking empty filepaths") logger.info("Done syncing library with directory: %s" % dir) # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select("SELECT ArtistName, ArtistID from artists") artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info("Updating track counts") for artist in current_artists: havetracks = len( myDB.select( "SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL", [artist["ArtistID"]] ) ) + len(myDB.select("SELECT TrackTitle from have WHERE ArtistName like ?", [artist["ArtistName"]])) myDB.action("UPDATE artists SET HaveTracks=? WHERE ArtistID=?", [havetracks, artist["ArtistID"]]) logger.info("Found %i new artists" % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info("Importing %i new artists" % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info("To add these artists, go to Manage->Manage New Artists") headphones.NEW_ARTISTS = artist_list if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000
def get_new_releases(rgid,includeExtras=False,forcefull=False): myDB = db.DBConnection() results = [] try: limit = 100 newResults = None while newResults == None or len(newResults) >= limit: newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) if 'release-list' not in newResults: break #may want to raise an exception here instead ? newResults = newResults['release-list'] results += newResults except musicbrainzngs.WebServiceError as e: logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) time.sleep(5) return False if not results or len(results) == 0: return False #Clean all references to releases in dB that are no longer referenced in musicbrainz release_list = [] force_repackage1 = 0 if len(results) != 0: for release_mark in results: release_list.append(unicode(release_mark['id'])) release_title = release_mark['title'] remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) if remove_missing_releases: for items in remove_missing_releases: if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: # Remove all from albums/tracks that aren't in release myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) force_repackage1 = 1 else: logger.info("There was either an error pulling data from MusicBrainz or there might not be any releases for this category") num_new_releases = 0 for releasedata in results: #releasedata.get will return None if it doesn't have a status #all official releases should have the Official status included if not includeExtras and releasedata.get('status') != 'Official': continue release = {} rel_id_check = releasedata['id'] artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() if not album_checker or forcefull: #DELETE all references to this release since we're updating it anyway. myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check]) myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check]) release['AlbumTitle'] = unicode(releasedata['title']) release['AlbumID'] = unicode(rgid) release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None release['ReleaseID'] = releasedata['id'] if 'release-group' not in releasedata: raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) release['Type'] = unicode(releasedata['release-group']['type']) if release['Type'] == 'Album' and 'secondary-type-list' in releasedata['release-group']: secondary_type = unicode(releasedata['release-group']['secondary-type-list'][0]) if secondary_type != release['Type']: release['Type'] = secondary_type #making the assumption that the most important artist will be first in the list if 'artist-credit' in releasedata: release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) else: logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') return False release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' #assuming that the list will contain media and that the format will be consistent try: additional_medium='' for position in releasedata['medium-list']: if position['format'] == releasedata['medium-list'][0]['format']: medium_count = int(position['position']) else: additional_medium = additional_medium+' + '+position['format'] if medium_count == 1: disc_number = '' else: disc_number = str(medium_count)+'x' packaged_medium = disc_number+releasedata['medium-list'][0]['format']+additional_medium release['ReleaseFormat'] = unicode(packaged_medium) except: release['ReleaseFormat'] = u'Unknown' release['Tracks'] = getTracksFromRelease(releasedata) # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table controlValueDict = {"ReleaseID" : release['ReleaseID']} newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "ReleaseDate": release['ReleaseDate'], "Type": release['Type'], "ReleaseCountry": release['ReleaseCountry'], "ReleaseFormat": release['ReleaseFormat'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in release['Tracks']: cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": release['ReleaseID']} newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() #if not match: #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) num_new_releases = num_new_releases + 1 #print releasedata['title'] #print num_new_releases if album_checker: logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check)) else: logger.info('[%s] New release %s (%s) added' % (release['ArtistName'], release['AlbumTitle'], rel_id_check)) if force_repackage1 == 1: num_new_releases = -1 logger.info('[%s] Forcing repackage of %s, since dB releases have been removed' % (release['ArtistName'], release_title)) else: num_new_releases = num_new_releases return num_new_releases
def libraryScan(dir=None): if not dir: dir = headphones.MUSIC_DIR try: dir = str(dir) except UnicodeEncodeError: dir = unicode(dir).encode('unicode_escape') if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir) return myDB = db.DBConnection() # Clean up bad filepaths tracks = myDB.select( 'SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode( headphones.SYS_ENCODING)): myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) logger.info('Scanning music directory: %s' % dir) new_artists = [] bitrates = [] myDB.action('DELETE from have') for r, d, f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors='replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + file) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Try to find a match based on artist/album/tracktitle if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: continue if f_artist and f.album and f.title: track = myDB.action( 'SELECT TrackID from tracks WHERE CleanName LIKE ?', [ helpers.cleanName(f_artist + ' ' + f.album + ' ' + f.title) ]).fetchone() if not track: track = myDB.action( 'SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() if track: myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']]) continue # Try to match on mbid if available and we couldn't find a match based on metadata if f.mb_trackid: # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action( 'SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() if track: myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']]) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release new_artists.append(f_artist) # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database myDB.action( 'INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [ f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist + ' ' + f.album + ' ' + f.title), f.format ]) logger.info('Completed scanning of directory: %s' % dir) logger.info('Checking filepaths to see if we can find any matches') # Now check empty file paths to see if we can find a match based on their folder format tracks = myDB.select('SELECT * from tracks WHERE Location IS NULL') for track in tracks: release = myDB.action('SELECT * from albums WHERE AlbumID=?', [track['AlbumID']]).fetchone() try: year = release['ReleaseDate'][:4] except TypeError: year = '' artist = release['ArtistName'].replace('/', '_') album = release['AlbumTitle'].replace('/', '_') releasetype = release['Type'].replace('/', '_') if release['ArtistName'].startswith('The '): sortname = release['ArtistName'][4:] else: sortname = release['ArtistName'] if sortname.isdigit(): firstchar = '0-9' else: firstchar = sortname[0] albumvalues = { '$Artist': artist, '$Album': album, '$Year': year, '$Type': releasetype, '$First': firstchar, '$artist': artist.lower(), '$album': album.lower(), '$year': year, '$type': releasetype.lower(), '$first': firstchar.lower() } folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues) folder = folder.replace('./', '_/').replace(':', '_').replace('?', '_') if folder.endswith('.'): folder = folder.replace(folder[len(folder) - 1], '_') if not track['TrackNumber']: tracknumber = '' else: tracknumber = '%02d' % track['TrackNumber'] title = track['TrackTitle'] trackvalues = { '$Track': tracknumber, '$Title': title, '$Artist': release['ArtistName'], '$Album': release['AlbumTitle'], '$Year': year, '$track': tracknumber, '$title': title.lower(), '$artist': release['ArtistName'].lower(), '$album': release['AlbumTitle'].lower(), '$year': year } new_file_name = helpers.replace_all( headphones.FILE_FORMAT, trackvalues).replace('/', '_') + '.*' new_file_name = new_file_name.replace('?', '_').replace(':', '_') full_path_to_file = os.path.normpath( os.path.join(headphones.MUSIC_DIR, folder, new_file_name)).encode(headphones.SYS_ENCODING, 'replace') match = glob.glob(full_path_to_file) if match: logger.info('Found a match: %s. Writing MBID to metadata' % match[0]) unipath = unicode(match[0], headphones.SYS_ENCODING, errors='replace') myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [unipath, track['TrackID']]) myDB.action('DELETE from have WHERE Location=?', [unipath]) # Try to insert the appropriate track id so we don't have to keep doing this try: f = MediaFile(match[0]) f.mb_trackid = track['TrackID'] f.save() myDB.action( 'UPDATE tracks SET BitRate=?, Format=? WHERE TrackID=?', [f.bitrate, f.format, track['TrackID']]) logger.debug('Wrote mbid to track: %s' % match[0]) except: logger.error('Error embedding track id into: %s' % match[0]) continue logger.info('Done checking empty filepaths') logger.info('Done syncing library with directory: %s' % dir) # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [ f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists] ] # Update track counts logger.info('Updating track counts') for artist in current_artists: havetracks = len( myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len( myDB.select( 'SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info( 'To add these artists, go to Manage->Manage New Artists') headphones.NEW_ARTISTS = artist_list if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000