def test_clean_name(self): """helpers: check correctness of clean_name() function""" cases = { u' Weiße & rose ': 'Weisse and rose', u'Multiple / spaces': 'Multiple spaces', u'Kevin\'s m²': 'Kevins m2', u'Symphonęy Nº9': 'Symphoney No.9', u'ÆæßðÞIJij': u'AeaessdThIJıj', u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral ' 'apoplexy remix', u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu ' 'zbojow', u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar', u'Björk Guðmundsdóttir': 'bjork gudmundsdottir', u'L\'Arc~en~Ciel': 'larc en ciel', u'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)': u'Orquesta de la Luz オルケスタ デ ラ ルス' } for first, second in cases.iteritems(): nf = clean_name(first).lower() ns = clean_name(second).lower() self.assertEqual(nf, ns, u"check cleaning of case (%s," u"%s)" % (nf, ns))
def test_clean_name(self): """helpers: check correctness of clean_name() function""" cases = { u' Weiße & rose ': 'Weisse and rose', u'Multiple / spaces': 'Multiple spaces', u'Kevin\'s m²': 'Kevins m2', u'Symphonęy Nº9': 'Symphoney No.9', u'ÆæßðÞIJij': u'AeaessdThIJıj', u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral ' 'apoplexy remix', u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu ' 'zbojow', u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar', u'Björk Guðmundsdóttir': 'bjork gudmundsdottir', u'L\'Arc~en~Ciel': 'larc en ciel', u'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)': u'Orquesta de la Luz オルケスタ デ ラ ルス' } for first, second in cases.iteritems(): nf = clean_name(first).lower() ns = clean_name(second).lower() self.assertEqual( nf, ns, u"check cleaning of case (%s," u"%s)" % (nf, ns) )
def test_clean_name_nonunicode(self): """helpers: check if clean_name() works on non-unicode input""" input = 'foo $ bar/BAZ' test = clean_name(input).lower() expected = 'foo bar baz' self.assertEqual(test, expected, "check clean_name() works on non-unicode") input = 'fóó $ BAZ' test = clean_name(input).lower() expected = clean_name('%fóó baz ').lower() self.assertEqual(test, expected, "check clean_name() with narrow non-ascii input")
def test_clean_name_nonunicode(self): """helpers: check if clean_name() works on non-unicode input""" input = 'foo $ bar/BAZ' test = clean_name(input).lower() expected = 'foo bar baz' self.assertEqual( test, expected, "check clean_name() works on non-unicode" ) input = 'fóó $ BAZ' test = clean_name(input).lower() expected = clean_name('%fóó baz ').lower() self.assertEqual( test, expected, "check clean_name() with narrow non-ascii input" )
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False, artistScan=False): if cron and not headphones.CONFIG.LIBRARYSCAN: return if not dir: if not headphones.CONFIG.MUSIC_DIR: return else: dir = headphones.CONFIG.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append or artistScan: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace')) return myDB = db.DBConnection() new_artists = [] logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up bad filepaths. Queries can take some time, ensure all results are loaded before processing if ArtistID: tracks = myDB.action( 'SELECT Location FROM alltracks WHERE ArtistID = ? AND Location IS NOT NULL UNION SELECT Location FROM tracks WHERE ArtistID = ? AND Location ' 'IS NOT NULL', [ArtistID, ArtistID]) else: tracks = myDB.action( 'SELECT Location FROM alltracks WHERE Location IS NOT NULL UNION SELECT Location FROM tracks WHERE Location IS NOT NULL' ) locations = [] for track in tracks: locations.append(track['Location']) for location in locations: encoded_track_string = location.encode(headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, location]) myDB.action( 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, location]) if ArtistName: del_have_tracks = myDB.select( 'SELECT Location, Matched, ArtistName FROM have WHERE ArtistName = ? COLLATE NOCASE', [ArtistName]) else: del_have_tracks = myDB.select( 'SELECT Location, Matched, ArtistName FROM have') locations = [] for track in del_have_tracks: locations.append([track['Location'], track['ArtistName']]) for location in locations: encoded_track_string = location[0].encode(headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): if location[1]: # Make sure deleted files get accounted for when updating artist track counts new_artists.append(location[1]) myDB.action('DELETE FROM have WHERE Location=?', [location[0]]) logger.info( 'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) bitrates = [] song_list = [] latest_subdirectory = [] new_song_count = 0 file_count = 0 for r, d, f in helpers.walk_directory(dir): # Filter paths based on config. Note that these methods work directly # on the inputs helpers.path_filter_patterns(d, headphones.CONFIG.IGNORED_FOLDERS, r) helpers.path_filter_patterns(f, headphones.CONFIG.IGNORED_FILES, r) for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): subdirectory = r.replace(dir, '') latest_subdirectory.append(subdirectory) if file_count == 0 and r.replace(dir, '') != '': logger.info( "[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) elif latest_subdirectory[file_count] != latest_subdirectory[ file_count - 1] and file_count != 0: logger.info( "[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except (FileTypeError, UnreadableFileError): logger.warning( "Cannot read media file '%s', skipping. It may be corrupted or not a media file.", unicode_song_path) continue except IOError: logger.warning( "Cannnot read media file '%s', skipping. Does the file exists?", unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) if f_artist and f.album and f.title: CleanName = helpers.clean_name(f_artist + ' ' + f.album + ' ' + f.title) else: CleanName = None controlValueDict = {'Location': unicode_song_path} newValueDict = { 'TrackID': f.mb_trackid, # 'ReleaseID' : f.mb_albumid, 'ArtistName': f_artist, 'AlbumTitle': f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre': f.genre, 'Date': f.date, 'TrackTitle': f.title, 'BitRate': f.bitrate, 'Format': f.format, 'CleanName': CleanName } # song_list.append(song_dict) check_exist_song = myDB.action( "SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. if not check_exist_song: # This is a new track if f_artist: new_artists.append(f_artist) myDB.upsert("have", newValueDict, controlValueDict) new_song_count += 1 else: if check_exist_song[ 'ArtistName'] != f_artist or check_exist_song[ 'AlbumTitle'] != f.album or check_exist_song[ 'TrackTitle'] != f.title: # Important track metadata has been modified, need to run matcher again if f_artist and f_artist != check_exist_song[ 'ArtistName']: new_artists.append(f_artist) elif f_artist and f_artist == check_exist_song['ArtistName'] and \ check_exist_song['Matched'] != "Ignored": new_artists.append(f_artist) else: continue newValueDict['Matched'] = None myDB.upsert("have", newValueDict, controlValueDict) myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) myDB.action( 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) new_song_count += 1 else: # This track information hasn't changed if f_artist and check_exist_song[ 'Matched'] != "Ignored": new_artists.append(f_artist) file_count += 1 # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) song_list = myDB.action( "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]) total_number_of_songs = \ myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0] logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches # song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 latest_artist = [] last_completion_percentage = 0 prev_artist_name = None artistid = None for song in song_list: latest_artist.append(song['ArtistName']) if song_count == 0: logger.info("Now matching songs by %s" % song['ArtistName']) elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0: logger.info("Now matching songs by %s" % song['ArtistName']) song_count += 1 completion_percentage = math.floor( float(song_count) / total_number_of_songs * 1000) / 10 if completion_percentage >= (last_completion_percentage + 10): logger.info("Track matching is " + str(completion_percentage) + "% complete") last_completion_percentage = completion_percentage # THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT # EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. albumid = None if song['ArtistName'] and song['CleanName']: artist_name = song['ArtistName'] clean_name = song['CleanName'] # Only update if artist is in the db if artist_name != prev_artist_name: prev_artist_name = artist_name artistid = None artist_lookup = "\"" + artist_name.replace("\"", "\"\"") + "\"" try: dbartist = myDB.select( 'SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '') except: dbartist = None if not dbartist: dbartist = myDB.select( 'SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName = ?', [clean_name]) if not dbartist: dbartist = myDB.select( 'SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName = ?', [clean_name]) if not dbartist: clean_artist = helpers.clean_name(artist_name) if clean_artist: dbartist = myDB.select( 'SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?', [clean_artist, clean_artist + '{']) if not dbartist: dbartist = myDB.select( 'SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?', [clean_artist, clean_artist + '{']) if dbartist: artistid = dbartist[0][0] if artistid: # This was previously using Artist, Album, Title with a SELECT LIKE ? and was not using an index # (Possible issue: https://stackoverflow.com/questions/37845854/python-sqlite3-not-using-index-with-like) # Now selects/updates using CleanName index (may have to revert if not working) # matching on CleanName should be enough, ensure it's the same artist just in case # Update tracks track = myDB.action( 'SELECT AlbumID, ArtistName FROM tracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone() if track: albumid = track['AlbumID'] myDB.action( 'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?', [ song['Location'], song['BitRate'], song['Format'], clean_name, artistid ]) # Update alltracks alltrack = myDB.action( 'SELECT AlbumID, ArtistName FROM alltracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone() if alltrack: albumid = alltrack['AlbumID'] myDB.action( 'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?', [ song['Location'], song['BitRate'], song['Format'], clean_name, artistid ]) # Update have controlValueDict2 = {'Location': song['Location']} if albumid: newValueDict2 = {'Matched': albumid} else: newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append or artistScan: logger.info('Updating scanned artist track counts') # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() # # Don't think we need to do this, check the db instead below # # # artist scan # if ArtistName: # current_artists = [[ArtistName]] # # directory scan # else: # current_artists = myDB.select('SELECT ArtistName, ArtistID FROM artists WHERE ArtistName IS NOT NULL') # # # There was a bug where artists with special characters (-,') would show up in new artists. # # # artist_list = scanned artists not in the db # artist_list = [ # x for x in unique_artists # if helpers.clean_name(x).lower() not in [ # helpers.clean_name(y[0]).lower() # for y in current_artists # ] # ] # # # artists_checked = scanned artists that exist in the db # artists_checked = [ # x for x in unique_artists # if helpers.clean_name(x).lower() in [ # helpers.clean_name(y[0]).lower() # for y in current_artists # ] # ] new_artist_list = [] for artist in unique_artists: if not artist: continue logger.info('Processing artist: %s' % artist) # check if artist is already in the db artist_lookup = "\"" + artist.replace("\"", "\"\"") + "\"" try: dbartist = myDB.select( 'SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '') except: dbartist = None if not dbartist: clean_artist = helpers.clean_name(artist) if clean_artist: dbartist = myDB.select( 'SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?', [clean_artist, clean_artist + '{']) if not dbartist: dbartist = myDB.select( 'SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?', [clean_artist, clean_artist + '{']) # new artist not in db, add to list if not dbartist: new_artist_list.append(artist) else: # artist in db, update have track counts artistid = dbartist[0][0] # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this # havetracks = ( # len(myDB.select( # 'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', # [artist])) + len(myDB.select( # 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', # [artist])) # ) try: havetracks = (len( myDB.select( 'SELECT ArtistID From tracks WHERE ArtistID = ? AND Location IS NOT NULL', [artistid]) ) + len( myDB.select( 'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))) except Exception as e: logger.warn('Error updating counts for artist: %s: %s' % (artist, e)) # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases # (can fix by getting rid of second len statement) if havetracks: myDB.action( 'UPDATE artists SET HaveTracks = ? WHERE ArtistID = ?', [havetracks, artistid]) # Update albums to downloaded update_album_status(ArtistID=artistid) logger.info('Found %i new artists' % len(new_artist_list)) # Add scanned artists not in the db if new_artist_list: if headphones.CONFIG.AUTO_ADD_ARTISTS: logger.info('Importing %i new artists' % len(new_artist_list)) importer.artistlist_to_mbids(new_artist_list) else: logger.info( 'To add these artists, go to Manage->Manage New Artists') # myDB.action('DELETE from newartists') for artist in new_artist_list: myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) if headphones.CONFIG.DETECT_BITRATE and bitrates: headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len( bitrates) / 1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') artist_lookup = "\"" + ArtistName.replace("\"", "\"\"") + "\"" try: havetracks = len( myDB.select( 'SELECT ArtistID FROM tracks WHERE ArtistID = ? AND Location IS NOT NULL', [ArtistID]) ) + len( myDB.select( 'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"')) except Exception as e: logger.warn('Error updating counts for artist: %s: %s' % (ArtistName, e)) if havetracks: myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) # Moved above to call for each artist # if not append: # update_album_status() if not append and not artistScan: lastfm.getSimilar() if ArtistName: logger.info('Scanning complete for artist: %s', ArtistName) else: logger.info('Library scan complete')
def addReleaseById(rid, rgid=None): myDB = db.DBConnection() # Create minimum info upfront if added from searchresults status = '' if rgid: dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) if not dbalbum: status = 'Loading' controlValueDict = {"AlbumID": rgid} newValueDict = {"AlbumTitle": rgid, "ArtistName": status, "Status": status} myDB.upsert("albums", newValueDict, controlValueDict) time.sleep(1) rgid = None artistid = None release_dict = None results = myDB.select( "SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid]) for result in results: rgid = result['ReleaseGroupID'] artistid = result['ArtistID'] logger.debug( "Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid) if not rgid: # didn't find it in the cache, get the information from MB logger.debug( "Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID") try: release_dict = mb.getRelease(rid) except Exception as e: logger.info('Unable to get release information for Release %s: %s', rid, e) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not release_dict: logger.info('Unable to get release information for Release %s: no dict', rid) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return rgid = release_dict['rgid'] artistid = release_dict['artist_id'] # we don't want to make more calls to MB here unless we have to, could be happening quite a lot rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) # make sure the artist exists since I don't know what happens later if it doesn't artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) if not artist_exists and release_dict: if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: sortname = release_dict['artist_name'] logger.info( u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused") controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = {"ArtistName": release_dict['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused"} if headphones.CONFIG.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 newValueDict['Extras'] = headphones.CONFIG.EXTRAS if 'title' in release_dict: newValueDict['LatestAlbum'] = release_dict['title'] elif 'rg_title' in release_dict: newValueDict['LatestAlbum'] = release_dict['rg_title'] if 'date' in release_dict: newValueDict['ReleaseDate'] = release_dict['date'] if rgid: newValueDict['AlbumID'] = rgid myDB.upsert("artists", newValueDict, controlValueDict) elif not artist_exists and not release_dict: logger.error( "Artist does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not rg_exists and release_dict or status == 'Loading' and release_dict: # it should never be the case that we have an rg and not the artist # but if it is this will fail logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid) controlValueDict = {"AlbumID": rgid} if status != 'Loading': status = 'Wanted' newValueDict = {"ArtistID": release_dict['artist_id'], "ReleaseID": rgid, "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'] if 'title' in release_dict else release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['date'], "DateAdded": helpers.today(), "Status": status, "Type": release_dict['rg_type'], "ReleaseID": rid } myDB.upsert("albums", newValueDict, controlValueDict) # keep a local cache of these so that external programs that are adding releasesByID don't hammer MB myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: cleanname = helpers.clean_name( release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "AlbumID": rgid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action( 'SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] # myDB.action('DELETE from have WHERE Location=?', [match['Location']]) # If the album has been scanned before adding the release it will be unmatched, update to matched if match['Matched'] == 'Failed': myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release_dict['rgid'], match['Location'])) myDB.upsert("tracks", newValueDict, controlValueDict) # Reset status if status == 'Loading': controlValueDict = {"AlbumID": rgid} if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: newValueDict = {"Status": "Wanted"} else: newValueDict = {"Status": "Skipped"} myDB.upsert("albums", newValueDict, controlValueDict) # Start a search for the album if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: import searcher searcher.searchforalbum(rgid, False) elif not rg_exists and not release_dict: logger.error( "ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return else: logger.info('Release ' + str(rid) + " already exists in the database!")
def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn('Cannot import blocked special purpose artist with id' + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS, "Extras": headphones.CONFIG.EXTRAS} if type == "series": newValueDict['Type'] = "series" else: newValueDict = {"Status": "Loading"} if dbartist["Type"] == "series": type = "series" myDB.upsert("artists", newValueDict, controlValueDict) if type == "series": artist = mb.getSeries(artistid) else: artist = mb.getArtist(artistid, extrasonly) if artist and artist.get('artist_name') in blacklisted_special_artist_names: logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name')) myDB.action('DELETE from artists WHERE ArtistID=?', [artistid]) # in case it's already in the db myDB.action('DELETE from albums WHERE ArtistID=?', [artistid]) myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence # over global option. Global options are set when adding a new artist try: db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False # Clean all references to release group in dB that are no longer referenced # from the musicbrainz refresh group_list = [] force_repackage = 0 # Don't nuke the database if there's a MusicBrainz error if len(artist['releasegroups']) != 0: for groups in artist['releasegroups']: group_list.append(groups['id']) if not extrasonly: remove_missing_groups_from_albums = myDB.select( "SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) else: remove_missing_groups_from_albums = myDB.select( 'SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid]) for items in remove_missing_groups_from_albums: if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']]) logger.info( "[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % ( artist['artist_name'], items['AlbumID'])) if not extrasonly: force_repackage = 1 else: if not extrasonly: logger.info( "[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name']) # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks album_searches = [] for rg in artist['releasegroups']: al_title = rg['title'] today = helpers.today() rgid = rg['id'] skip_log = 0 # Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.CONFIG.MB_IGNORE_AGE rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() if not forcefull: new_release_group = False try: check_release_date = rg_exists['ReleaseDate'] except TypeError: check_release_date = None new_release_group = True if new_release_group: logger.info("[%s] Now adding: %s (New Release Group)" % ( artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid, includeExtras) else: if check_release_date is None or check_release_date == u"None": logger.info("[%s] Now updating: %s (No Release Date)" % ( artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: if len(check_release_date) == 10: release_date = check_release_date elif len(check_release_date) == 7: release_date = check_release_date + "-31" elif len(check_release_date) == 4: release_date = check_release_date + "-12-31" else: release_date = today if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta: logger.info("[%s] Now updating: %s (Release Date <%s Days)", artist['artist_name'], rg['title'], pause_delta) new_releases = mb.get_new_releases(rgid, includeExtras, True) else: logger.info("[%s] Skipping: %s (Release Date >%s Days)", artist['artist_name'], rg['title'], pause_delta) skip_log = 1 new_releases = 0 if force_repackage == 1: new_releases = -1 logger.info('[%s] Forcing repackage of %s (Release Group Removed)', artist['artist_name'], al_title) else: new_releases = new_releases else: logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'], rg['title']) new_releases = mb.get_new_releases(rgid, includeExtras, forcefull) if new_releases != 0: # Dump existing hybrid release since we're repackaging/replacing it myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rg['id']]) # This will be used later to build a hybrid release fullreleaselist = [] # Search for releases within a release group find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) # Build the dictionary for the fullreleaselist for items in find_hybrid_releases: if items['ReleaseID'] != rg[ 'id']: # don't include hybrid information, since that's what we're replacing hybrid_release_id = items['ReleaseID'] newValueDict = {"ArtistID": items['ArtistID'], "ArtistName": items['ArtistName'], "AlbumTitle": items['AlbumTitle'], "AlbumID": items['AlbumID'], "AlbumASIN": items['AlbumASIN'], "ReleaseDate": items['ReleaseDate'], "Type": items['Type'], "ReleaseCountry": items['ReleaseCountry'], "ReleaseFormat": items['ReleaseFormat'] } find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] for hybrid_tracks in find_hybrid_tracks: hybrid_track_array.append({ 'number': hybrid_tracks['TrackNumber'], 'title': hybrid_tracks['TrackTitle'], 'id': hybrid_tracks['TrackID'], # 'url': hybrid_tracks['TrackURL'], 'duration': hybrid_tracks['TrackDuration'] }) totalTracks += 1 newValueDict['ReleaseID'] = hybrid_release_id newValueDict['Tracks'] = hybrid_track_array fullreleaselist.append(newValueDict) # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) logger.info('[%s] Packaging %s releases into hybrid title' % ( artist['artist_name'], rg['title'])) except Exception as e: errors = True logger.warn('[%s] Unable to get hybrid release information for %s: %s' % ( artist['artist_name'], rg['title'], e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.clean_name( artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "ReleaseID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table # myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) # check if the album already exists if not rg_exists: releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] if not releaseid: releaseid = rg['id'] album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if rg_exists: newValueDict['DateAdded'] = rg_exists['DateAdded'] newValueDict['Status'] = rg_exists['Status'] else: today = helpers.today() newValueDict['DateAdded'] = today if headphones.CONFIG.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif helpers.get_age(album['ReleaseDate']) and helpers.get_age( today) - helpers.get_age( album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to # mark it as downloaded. Default is 80%, can be set in config as # ALBUM_COMPLETION_PCT total_track_count = len(tracks) if total_track_count == 0: logger.warning("Total track count is zero for Release ID " + "'%s', skipping.", releaseid) continue for track in tracks: controlValueDict = {"TrackID": track['TrackID'], "AlbumID": rg['id']} newValueDict = {"ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len( myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) marked_as_downloaded = False if rg_exists: if rg_exists['Status'] == 'Skipped' and ( (have_track_count / float(total_track_count)) >= ( headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True else: if (have_track_count / float(total_track_count)) >= ( headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True logger.info( u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title'])) cache.getThumb(AlbumID=rg['id']) # Start a search for the album if it's new, hasn't been marked as # downloaded and autowant_all is selected. This search is deferred, # in case the search failes and the rest of the import will halt. if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL: album_searches.append(rg['id']) else: if skip_log == 0: logger.info(u"[%s] No new releases, so no changes made to %s" % ( artist['artist_name'], rg['title'])) time.sleep(3) finalize_update(artistid, artist['artist_name'], errors) logger.info(u"Seeing if we need album art for: %s" % artist['artist_name']) cache.getThumb(ArtistID=artistid) logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name']) metacritic.update(artistid, artist['artist_name'], artist['releasegroups']) if errors: logger.info( "[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % ( artist['artist_name'], artist['artist_name'])) else: myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']]) logger.info(u"Updating complete for: %s" % artist['artist_name']) # Start searching for newly added albums if album_searches: from headphones import searcher logger.info("Start searching for %d albums.", len(album_searches)) for album_search in album_searches: searcher.searchforalbum(albumid=album_search)
def addReleaseById(rid, rgid=None): myDB = db.DBConnection() # Create minimum info upfront if added from searchresults status = '' if rgid: dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) if not dbalbum: status = 'Loading' controlValueDict = {"AlbumID": rgid} newValueDict = { "AlbumTitle": rgid, "ArtistName": status, "Status": status } myDB.upsert("albums", newValueDict, controlValueDict) time.sleep(1) rgid = None artistid = None release_dict = None results = myDB.select( "SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid]) for result in results: rgid = result['ReleaseGroupID'] artistid = result['ArtistID'] logger.debug( "Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid) if not rgid: # didn't find it in the cache, get the information from MB logger.debug("Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID") try: release_dict = mb.getRelease(rid) except Exception as e: logger.info('Unable to get release information for Release %s: %s', rid, e) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not release_dict: logger.info( 'Unable to get release information for Release %s: no dict', rid) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return rgid = release_dict['rgid'] artistid = release_dict['artist_id'] # we don't want to make more calls to MB here unless we have to, could be happening quite a lot rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) # make sure the artist exists since I don't know what happens later if it doesn't artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) if not artist_exists and release_dict: if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: sortname = release_dict['artist_name'] logger.info(u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused") controlValueDict = {"ArtistID": release_dict['artist_id']} newValueDict = { "ArtistName": release_dict['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Paused" } if headphones.CONFIG.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 newValueDict['Extras'] = headphones.CONFIG.EXTRAS if 'title' in release_dict: newValueDict['LatestAlbum'] = release_dict['title'] elif 'rg_title' in release_dict: newValueDict['LatestAlbum'] = release_dict['rg_title'] if 'date' in release_dict: newValueDict['ReleaseDate'] = release_dict['date'] if rgid: newValueDict['AlbumID'] = rgid myDB.upsert("artists", newValueDict, controlValueDict) elif not artist_exists and not release_dict: logger.error( "Artist does not exist in the database and did not get a valid response from MB. Skipping release." ) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return if not rg_exists and release_dict or status == 'Loading' and release_dict: # it should never be the case that we have an rg and not the artist # but if it is this will fail logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid) controlValueDict = {"AlbumID": rgid} if status != 'Loading': status = 'Wanted' newValueDict = { "ArtistID": release_dict['artist_id'], "ReleaseID": rgid, "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'] if 'title' in release_dict else release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "ReleaseDate": release_dict['date'], "DateAdded": helpers.today(), "Status": status, "Type": release_dict['rg_type'], "ReleaseID": rid } myDB.upsert("albums", newValueDict, controlValueDict) # keep a local cache of these so that external programs that are adding releasesByID don't hammer MB myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: cleanname = helpers.clean_name(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], "AlbumID": rgid} newValueDict = { "ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['rg_title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action( 'SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [ release_dict['artist_name'], release_dict['rg_title'], track['title'] ]).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] # myDB.action('DELETE from have WHERE Location=?', [match['Location']]) # If the album has been scanned before adding the release it will be unmatched, update to matched if match['Matched'] == 'Failed': myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release_dict['rgid'], match['Location'])) myDB.upsert("tracks", newValueDict, controlValueDict) # Reset status if status == 'Loading': controlValueDict = {"AlbumID": rgid} if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: newValueDict = {"Status": "Wanted"} else: newValueDict = {"Status": "Skipped"} myDB.upsert("albums", newValueDict, controlValueDict) # Start a search for the album if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: import searcher searcher.searchforalbum(rgid, False) elif not rg_exists and not release_dict: logger.error( "ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release." ) if status == 'Loading': myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) return else: logger.info('Release ' + str(rid) + " already exists in the database!")
def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache # Can't add various artists - throws an error from MB if artistid in blacklisted_special_artists: logger.warn('Cannot import blocked special purpose artist with id' + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp errors = False myDB = db.DBConnection() # Delete from blacklist if it's on there myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: newValueDict = { "ArtistName": "Artist ID: %s" % (artistid), "Status": "Loading", "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS, "Extras": headphones.CONFIG.EXTRAS } if type == "series": newValueDict['Type'] = "series" else: newValueDict = {"Status": "Loading"} if dbartist["Type"] == "series": type = "series" myDB.upsert("artists", newValueDict, controlValueDict) if type == "series": artist = mb.getSeries(artistid) else: artist = mb.getArtist(artistid, extrasonly) if artist and artist.get( 'artist_name') in blacklisted_special_artist_names: logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name')) myDB.action('DELETE from artists WHERE ArtistID=?', [artistid]) # in case it's already in the db myDB.action('DELETE from albums WHERE ArtistID=?', [artistid]) myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid]) return if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: newValueDict = { "ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active" } else: newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return if artist['artist_name'].startswith('The '): sortname = artist['artist_name'][4:] else: sortname = artist['artist_name'] logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = { "ArtistName": artist['artist_name'], "ArtistSortName": sortname, "DateAdded": helpers.today(), "Status": "Loading" } myDB.upsert("artists", newValueDict, controlValueDict) # See if we need to grab extras. Artist specific extras take precedence # over global option. Global options are set when adding a new artist try: db_artist = myDB.action( 'SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: includeExtras = False # Clean all references to release group in dB that are no longer referenced # from the musicbrainz refresh group_list = [] force_repackage = 0 # Don't nuke the database if there's a MusicBrainz error if len(artist['releasegroups']) != 0: for groups in artist['releasegroups']: group_list.append(groups['id']) if not extrasonly: remove_missing_groups_from_albums = myDB.select( "SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) else: remove_missing_groups_from_albums = myDB.select( 'SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid]) for items in remove_missing_groups_from_albums: if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']]) logger.info( "[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (artist['artist_name'], items['AlbumID'])) if not extrasonly: force_repackage = 1 else: if not extrasonly: logger.info( "[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name']) # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks album_searches = [] for rg in artist['releasegroups']: al_title = rg['title'] today = helpers.today() rgid = rg['id'] skip_log = 0 # Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.CONFIG.MB_IGNORE_AGE rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() if not forcefull: new_release_group = False try: check_release_date = rg_exists['ReleaseDate'] except TypeError: check_release_date = None new_release_group = True if new_release_group: logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases(rgid, includeExtras) else: if check_release_date is None or check_release_date == u"None": if headphones.CONFIG.MB_IGNORE_AGE_MISSING is not 1: logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title'])) new_releases = mb.get_new_releases( rgid, includeExtras, True) else: logger.info( "[%s] Skipping update of: %s (No Release Date)" % (artist['artist_name'], rg['title'])) new_releases = 0 else: if len(check_release_date) == 10: release_date = check_release_date elif len(check_release_date) == 7: release_date = check_release_date + "-31" elif len(check_release_date) == 4: release_date = check_release_date + "-12-31" else: release_date = today if helpers.get_age(today) - helpers.get_age( release_date) < pause_delta: logger.info( "[%s] Now updating: %s (Release Date <%s Days)", artist['artist_name'], rg['title'], pause_delta) new_releases = mb.get_new_releases( rgid, includeExtras, True) else: logger.info( "[%s] Skipping: %s (Release Date >%s Days)", artist['artist_name'], rg['title'], pause_delta) skip_log = 1 new_releases = 0 if force_repackage == 1: new_releases = -1 logger.info( '[%s] Forcing repackage of %s (Release Group Removed)', artist['artist_name'], al_title) else: new_releases = new_releases else: logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'], rg['title']) new_releases = mb.get_new_releases(rgid, includeExtras, forcefull) if new_releases != 0: # Dump existing hybrid release since we're repackaging/replacing it myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rg['id']]) # This will be used later to build a hybrid release fullreleaselist = [] # Search for releases within a release group find_hybrid_releases = myDB.action( "SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) # Build the dictionary for the fullreleaselist for items in find_hybrid_releases: # don't include hybrid information, since that's what we're replacing if items['ReleaseID'] != rg['id']: hybrid_release_id = items['ReleaseID'] newValueDict = { "ArtistID": items['ArtistID'], "ArtistName": items['ArtistName'], "AlbumTitle": items['AlbumTitle'], "AlbumID": items['AlbumID'], "AlbumASIN": items['AlbumASIN'], "ReleaseDate": items['ReleaseDate'], "Type": items['Type'], "ReleaseCountry": items['ReleaseCountry'], "ReleaseFormat": items['ReleaseFormat'] } find_hybrid_tracks = myDB.action( "SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] for hybrid_tracks in find_hybrid_tracks: hybrid_track_array.append({ 'number': hybrid_tracks['TrackNumber'], 'title': hybrid_tracks['TrackTitle'], 'id': hybrid_tracks['TrackID'], # 'url': hybrid_tracks['TrackURL'], 'duration': hybrid_tracks['TrackDuration'] }) totalTracks += 1 newValueDict['ReleaseID'] = hybrid_release_id newValueDict['Tracks'] = hybrid_track_array fullreleaselist.append(newValueDict) # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist try: hybridrelease = getHybridRelease(fullreleaselist) logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title'])) except Exception as e: errors = True logger.warn( '[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'], rg['title'], e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format controlValueDict = {"ReleaseID": rg['id']} newValueDict = { "ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumID": rg['id'], "AlbumASIN": hybridrelease['AlbumASIN'], "ReleaseDate": hybridrelease['ReleaseDate'], "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in hybridrelease['Tracks']: cleanname = helpers.clean_name(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = { "TrackID": track['id'], "ReleaseID": rg['id'] } newValueDict = { "ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], "AlbumASIN": hybridrelease['AlbumASIN'], "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title'] ]).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table # myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) # check if the album already exists if not rg_exists: releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] if not releaseid: releaseid = rg['id'] album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} newValueDict = { "ArtistID": album['ArtistID'], "ArtistName": album['ArtistName'], "AlbumTitle": album['AlbumTitle'], "ReleaseID": album['ReleaseID'], "AlbumASIN": album['AlbumASIN'], "ReleaseDate": album['ReleaseDate'], "Type": album['Type'], "ReleaseCountry": album['ReleaseCountry'], "ReleaseFormat": album['ReleaseFormat'] } if rg_exists: newValueDict['DateAdded'] = rg_exists['DateAdded'] newValueDict['Status'] = rg_exists['Status'] else: today = helpers.today() newValueDict['DateAdded'] = today if headphones.CONFIG.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif album[ 'ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day elif helpers.get_age( album['ReleaseDate'] ) and helpers.get_age(today) - helpers.get_age( album['ReleaseDate'] ) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" myDB.upsert("albums", newValueDict, controlValueDict) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to # mark it as downloaded. Default is 80%, can be set in config as # ALBUM_COMPLETION_PCT total_track_count = len(tracks) if total_track_count == 0: logger.warning( "Total track count is zero for Release ID " + "'%s', skipping.", releaseid) continue for track in tracks: controlValueDict = { "TrackID": track['TrackID'], "AlbumID": rg['id'] } newValueDict = { "ArtistID": track['ArtistID'], "ArtistName": track['ArtistName'], "AlbumTitle": track['AlbumTitle'], "AlbumASIN": track['AlbumASIN'], "ReleaseID": track['ReleaseID'], "TrackTitle": track['TrackTitle'], "TrackDuration": track['TrackDuration'], "TrackNumber": track['TrackNumber'], "CleanName": track['CleanName'], "Location": track['Location'], "Format": track['Format'], "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len( myDB.select( 'SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) marked_as_downloaded = False if rg_exists: if rg_exists['Status'] == 'Skipped' and ( (have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True else: if (have_track_count / float(total_track_count)) >= ( headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title'])) cache.getThumb(AlbumID=rg['id']) # Start a search for the album if it's new, hasn't been marked as # downloaded and autowant_all is selected. This search is deferred, # in case the search failes and the rest of the import will halt. if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL: album_searches.append(rg['id']) else: if skip_log == 0: logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title'])) time.sleep(3) finalize_update(artistid, artist['artist_name'], errors) logger.info(u"Seeing if we need album art for: %s" % artist['artist_name']) cache.getThumb(ArtistID=artistid) logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name']) metacritic.update(artistid, artist['artist_name'], artist['releasegroups']) if errors: logger.info( "[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name'])) else: myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']]) logger.info(u"Updating complete for: %s" % artist['artist_name']) # Start searching for newly added albums if album_searches: from headphones import searcher logger.info("Start searching for %d albums.", len(album_searches)) for album_search in album_searches: searcher.searchforalbum(albumid=album_search)
def get_new_releases(rgid, includeExtras=False, forcefull=False): myDB = db.DBConnection() results = [] release_status = "official" if includeExtras and not headphones.CONFIG.OFFICIAL_RELEASES_ONLY: release_status = [] try: limit = 100 newResults = None while newResults is None or len(newResults) >= limit: with mb_lock: newResults = musicbrainzngs.browse_releases( release_group=rgid, includes=[ 'artist-credits', 'labels', 'recordings', 'release-groups', 'media' ], release_status=release_status, limit=limit, offset=len(results)) if 'release-list' not in newResults: break # may want to raise an exception here instead ? newResults = newResults['release-list'] results += newResults except musicbrainzngs.WebServiceError as e: logger.warn( 'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) mb_lock.snooze(5) return False if not results or len(results) == 0: return False # Clean all references to releases in dB that are no longer referenced in musicbrainz release_list = [] force_repackage1 = 0 if len(results) != 0: for release_mark in results: release_list.append(unicode(release_mark['id'])) release_title = release_mark['title'] remove_missing_releases = myDB.action( "SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) if remove_missing_releases: for items in remove_missing_releases: if items['ReleaseID'] not in release_list and items[ 'ReleaseID'] != rgid: # Remove all from albums/tracks that aren't in release myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) logger.info( "Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) force_repackage1 = 1 else: logger.info( "There was either an error pulling data from MusicBrainz or there might not be any releases for this category" ) num_new_releases = 0 for releasedata in results: release = {} rel_id_check = releasedata['id'] album_checker = myDB.action( 'SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() if not album_checker or forcefull: # DELETE all references to this release since we're updating it anyway. myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check]) myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check]) release['AlbumTitle'] = unicode(releasedata['title']) release['AlbumID'] = unicode(rgid) release['AlbumASIN'] = unicode( releasedata['asin']) if 'asin' in releasedata else None release['ReleaseDate'] = unicode( releasedata['date']) if 'date' in releasedata else None release['ReleaseID'] = releasedata['id'] if 'release-group' not in releasedata: raise Exception( 'No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) release['Type'] = unicode(releasedata['release-group']['type']) if release[ 'Type'] == 'Album' and 'secondary-type-list' in releasedata[ 'release-group']: secondary_type = unicode( releasedata['release-group']['secondary-type-list'][0]) if secondary_type != release['Type']: release['Type'] = secondary_type # making the assumption that the most important artist will be first in the list if 'artist-credit' in releasedata: release['ArtistID'] = unicode( releasedata['artist-credit'][0]['artist']['id']) release['ArtistName'] = unicode( releasedata['artist-credit-phrase']) else: logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') return False release['ReleaseCountry'] = unicode( releasedata['country'] ) if 'country' in releasedata else u'Unknown' # assuming that the list will contain media and that the format will be consistent try: additional_medium = '' for position in releasedata['medium-list']: if position['format'] == releasedata['medium-list'][0][ 'format']: medium_count = int(position['position']) else: additional_medium = additional_medium + ' + ' + position[ 'format'] if medium_count == 1: disc_number = '' else: disc_number = str(medium_count) + 'x' packaged_medium = disc_number + releasedata['medium-list'][0][ 'format'] + additional_medium release['ReleaseFormat'] = unicode(packaged_medium) except: release['ReleaseFormat'] = u'Unknown' release['Tracks'] = getTracksFromRelease(releasedata) # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table controlValueDict = {"ReleaseID": release['ReleaseID']} newValueDict = { "ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "ReleaseDate": release['ReleaseDate'], "Type": release['Type'], "ReleaseCountry": release['ReleaseCountry'], "ReleaseFormat": release['ReleaseFormat'] } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in release['Tracks']: cleanname = helpers.clean_name(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) controlValueDict = { "TrackID": track['id'], "ReleaseID": release['ReleaseID'] } newValueDict = { "ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], "CleanName": cleanname } match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() if not match: match = myDB.action( 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [ release['ArtistName'], release['AlbumTitle'], track['title'] ]).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) num_new_releases = num_new_releases + 1 if album_checker: logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check)) else: logger.info('[%s] New release %s (%s) added' % (release['ArtistName'], release['AlbumTitle'], rel_id_check)) if force_repackage1 == 1: num_new_releases = -1 logger.info( '[%s] Forcing repackage of %s, since dB releases have been removed' % (release['ArtistName'], release_title)) else: num_new_releases = num_new_releases return num_new_releases
def get_new_releases(rgid, includeExtras=False, forcefull=False): myDB = db.DBConnection() results = [] release_status = "official" if includeExtras and not headphones.CONFIG.OFFICIAL_RELEASES_ONLY: release_status = [] try: limit = 100 newResults = None while newResults is None or len(newResults) >= limit: with mb_lock: newResults = musicbrainzngs.browse_releases( release_group=rgid, includes=["artist-credits", "labels", "recordings", "release-groups", "media"], release_status=release_status, limit=limit, offset=len(results), ) if "release-list" not in newResults: break # may want to raise an exception here instead ? newResults = newResults["release-list"] results += newResults except musicbrainzngs.WebServiceError as e: logger.warn( 'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e)) ) mb_lock.snooze(5) return False if not results or len(results) == 0: return False # Clean all references to releases in dB that are no longer referenced in musicbrainz release_list = [] force_repackage1 = 0 if len(results) != 0: for release_mark in results: release_list.append(unicode(release_mark["id"])) release_title = release_mark["title"] remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) if remove_missing_releases: for items in remove_missing_releases: if items["ReleaseID"] not in release_list and items["ReleaseID"] != rgid: # Remove all from albums/tracks that aren't in release myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items["ReleaseID"]]) myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items["ReleaseID"]]) myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items["ReleaseID"]]) myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items["ReleaseID"]]) logger.info("Removing all references to release %s to reflect MusicBrainz" % items["ReleaseID"]) force_repackage1 = 1 else: logger.info( "There was either an error pulling data from MusicBrainz or there might not be any releases for this category" ) num_new_releases = 0 for releasedata in results: release = {} rel_id_check = releasedata["id"] album_checker = myDB.action("SELECT * from allalbums WHERE ReleaseID=?", [rel_id_check]).fetchone() if not album_checker or forcefull: # DELETE all references to this release since we're updating it anyway. myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rel_id_check]) myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rel_id_check]) release["AlbumTitle"] = unicode(releasedata["title"]) release["AlbumID"] = unicode(rgid) release["AlbumASIN"] = unicode(releasedata["asin"]) if "asin" in releasedata else None release["ReleaseDate"] = unicode(releasedata["date"]) if "date" in releasedata else None release["ReleaseID"] = releasedata["id"] if "release-group" not in releasedata: raise Exception("No release group associated with release id " + releasedata["id"] + " album id" + rgid) release["Type"] = unicode(releasedata["release-group"]["type"]) if release["Type"] == "Album" and "secondary-type-list" in releasedata["release-group"]: secondary_type = unicode(releasedata["release-group"]["secondary-type-list"][0]) if secondary_type != release["Type"]: release["Type"] = secondary_type # making the assumption that the most important artist will be first in the list if "artist-credit" in releasedata: release["ArtistID"] = unicode(releasedata["artist-credit"][0]["artist"]["id"]) release["ArtistName"] = unicode(releasedata["artist-credit-phrase"]) else: logger.warn("Release " + releasedata["id"] + " has no Artists associated.") return False release["ReleaseCountry"] = unicode(releasedata["country"]) if "country" in releasedata else u"Unknown" # assuming that the list will contain media and that the format will be consistent try: additional_medium = "" for position in releasedata["medium-list"]: if position["format"] == releasedata["medium-list"][0]["format"]: medium_count = int(position["position"]) else: additional_medium = additional_medium + " + " + position["format"] if medium_count == 1: disc_number = "" else: disc_number = str(medium_count) + "x" packaged_medium = disc_number + releasedata["medium-list"][0]["format"] + additional_medium release["ReleaseFormat"] = unicode(packaged_medium) except: release["ReleaseFormat"] = u"Unknown" release["Tracks"] = getTracksFromRelease(releasedata) # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table controlValueDict = {"ReleaseID": release["ReleaseID"]} newValueDict = { "ArtistID": release["ArtistID"], "ArtistName": release["ArtistName"], "AlbumTitle": release["AlbumTitle"], "AlbumID": release["AlbumID"], "AlbumASIN": release["AlbumASIN"], "ReleaseDate": release["ReleaseDate"], "Type": release["Type"], "ReleaseCountry": release["ReleaseCountry"], "ReleaseFormat": release["ReleaseFormat"], } myDB.upsert("allalbums", newValueDict, controlValueDict) for track in release["Tracks"]: cleanname = helpers.clean_name( release["ArtistName"] + " " + release["AlbumTitle"] + " " + track["title"] ) controlValueDict = {"TrackID": track["id"], "ReleaseID": release["ReleaseID"]} newValueDict = { "ArtistID": release["ArtistID"], "ArtistName": release["ArtistName"], "AlbumTitle": release["AlbumTitle"], "AlbumID": release["AlbumID"], "AlbumASIN": release["AlbumASIN"], "TrackTitle": track["title"], "TrackDuration": track["duration"], "TrackNumber": track["number"], "CleanName": cleanname, } match = myDB.action( "SELECT Location, BitRate, Format from have WHERE CleanName=?", [cleanname] ).fetchone() if not match: match = myDB.action( "SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?", [release["ArtistName"], release["AlbumTitle"], track["title"]], ).fetchone() # if not match: # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict["Location"] = match["Location"] newValueDict["BitRate"] = match["BitRate"] newValueDict["Format"] = match["Format"] # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.action("UPDATE have SET Matched=? WHERE Location=?", (release["AlbumID"], match["Location"])) myDB.upsert("alltracks", newValueDict, controlValueDict) num_new_releases = num_new_releases + 1 if album_checker: logger.info( "[%s] Existing release %s (%s) updated" % (release["ArtistName"], release["AlbumTitle"], rel_id_check) ) else: logger.info( "[%s] New release %s (%s) added" % (release["ArtistName"], release["AlbumTitle"], rel_id_check) ) if force_repackage1 == 1: num_new_releases = -1 logger.info( "[%s] Forcing repackage of %s, since dB releases have been removed" % (release["ArtistName"], release_title) ) else: num_new_releases = num_new_releases return num_new_releases
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False, artistScan=False): if cron and not headphones.CONFIG.LIBRARYSCAN: return if not dir: if not headphones.CONFIG.MUSIC_DIR: return else: dir = headphones.CONFIG.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append or artistScan: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace')) return myDB = db.DBConnection() new_artists = [] logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up bad filepaths tracks = myDB.select( 'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL') for track in tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have') for track in del_have_tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): if track['ArtistName']: # Make sure deleted files get accounted for when updating artist track counts new_artists.append(track['ArtistName']) myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) logger.info( 'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode( headphones.SYS_ENCODING, 'replace')) bitrates = [] song_list = [] latest_subdirectory = [] new_song_count = 0 file_count = 0 for r, d, f in helpers.walk_directory(dir): # Filter paths based on config. Note that these methods work directly # on the inputs helpers.path_filter_patterns(d, headphones.CONFIG.IGNORED_FOLDERS, r) helpers.path_filter_patterns(f, headphones.CONFIG.IGNORED_FILES, r) for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): subdirectory = r.replace(dir, '') latest_subdirectory.append(subdirectory) if file_count == 0 and r.replace(dir, '') != '': logger.info("[%s] Now scanning subdirectory %s" % ( dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) elif latest_subdirectory[file_count] != latest_subdirectory[ file_count - 1] and file_count != 0: logger.info("[%s] Now scanning subdirectory %s" % ( dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except (FileTypeError, UnreadableFileError): logger.warning( "Cannot read media file '%s', skipping. It may be corrupted or not a media file.", unicode_song_path) continue except IOError: logger.warning("Cannnot read media file '%s', skipping. Does the file exists?", unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) if f_artist and f.album and f.title: CleanName = helpers.clean_name(f_artist + ' ' + f.album + ' ' + f.title) else: CleanName = None controlValueDict = {'Location': unicode_song_path} newValueDict = {'TrackID': f.mb_trackid, # 'ReleaseID' : f.mb_albumid, 'ArtistName': f_artist, 'AlbumTitle': f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre': f.genre, 'Date': f.date, 'TrackTitle': f.title, 'BitRate': f.bitrate, 'Format': f.format, 'CleanName': CleanName } # song_list.append(song_dict) check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. if not check_exist_song: # This is a new track if f_artist: new_artists.append(f_artist) myDB.upsert("have", newValueDict, controlValueDict) new_song_count += 1 else: if check_exist_song['ArtistName'] != f_artist or check_exist_song[ 'AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: # Important track metadata has been modified, need to run matcher again if f_artist and f_artist != check_exist_song['ArtistName']: new_artists.append(f_artist) elif f_artist and f_artist == check_exist_song['ArtistName'] and \ check_exist_song['Matched'] != "Ignored": new_artists.append(f_artist) else: continue newValueDict['Matched'] = None myDB.upsert("have", newValueDict, controlValueDict) myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) myDB.action( 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) new_song_count += 1 else: # This track information hasn't changed if f_artist and check_exist_song['Matched'] != "Ignored": new_artists.append(f_artist) file_count += 1 # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]) total_number_of_songs = \ myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0] logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode( headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches # song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 latest_artist = [] last_completion_percentage = 0 for song in song_list: latest_artist.append(song['ArtistName']) if song_count == 0: logger.info("Now matching songs by %s" % song['ArtistName']) elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0: logger.info("Now matching songs by %s" % song['ArtistName']) song_count += 1 completion_percentage = math.floor(float(song_count) / total_number_of_songs * 1000) / 10 if completion_percentage >= (last_completion_percentage + 10): logger.info("Track matching is " + str(completion_percentage) + "% complete") last_completion_percentage = completion_percentage # THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT # EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action( 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() have_updated = False if track: controlValueDict = {'ArtistName': track['ArtistName'], 'AlbumTitle': track['AlbumTitle'], 'TrackTitle': track['TrackTitle']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if track: controlValueDict = {'CleanName': track['CleanName']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True alltrack = myDB.action( 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if alltrack: controlValueDict = {'ArtistName': alltrack['ArtistName'], 'AlbumTitle': alltrack['AlbumTitle'], 'TrackTitle': alltrack['TrackTitle']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: alltrack = myDB.action( 'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if alltrack: controlValueDict = {'CleanName': alltrack['CleanName']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks if not have_updated: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) else: controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append or artistScan: logger.info('Updating scanned artist track counts') # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') # There was a bug where artists with special characters (-,') would show up in new artists. artist_list = [ x for x in unique_artists if helpers.clean_name(x).lower() not in [ helpers.clean_name(y[0]).lower() for y in current_artists ] ] artists_checked = [ x for x in unique_artists if helpers.clean_name(x).lower() in [ helpers.clean_name(y[0]).lower() for y in current_artists ] ] # Update track counts for artist in artists_checked: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = ( len(myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist])) + len(myDB.select( 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist])) ) # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases # (can fix by getting rid of second len statement) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) logger.info('Found %i new artists' % len(artist_list)) if artist_list: if headphones.CONFIG.AUTO_ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') # myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) if headphones.CONFIG.DETECT_BITRATE and bitrates: headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len( myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select( 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) if not append: update_album_status() if not append and not artistScan: lastfm.getSimilar() logger.info('Library scan complete')