def write(self): """Writes the item's metadata to the associated file. """ f = MediaFile(syspath(self.path)) for key in ITEM_KEYS_WRITABLE: setattr(f, key, getattr(self, key)) f.save()
def embedLyrics(downloaded_track_list): logger.info('Adding lyrics') for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except: logger.error('Could not read %s. Not checking lyrics' % downloaded_track) if f.albumartist and f.title: metalyrics = lyrics.getLyrics(f.albumartist, f.title) elif f.artist and f.title: metalyrics = lyrics.getLyrics(f.artist, f.title) else: logger.info( 'No artist/track metadata found for track: %s. Not fetching lyrics' % downloaded_track) metalyrics = None if lyrics: logger.debug('Adding lyrics to: %s' % downloaded_track) f.lyrics = metalyrics f.save()
def embedLyrics(downloaded_track_list): logger.info('Adding lyrics') # TODO: If adding lyrics for flac & lossy, only fetch the lyrics once # and apply it to both files for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except: logger.error( 'Could not read %s. Not checking lyrics' % downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) continue if f.albumartist and f.title: metalyrics = lyrics.getLyrics(f.albumartist, f.title) elif f.artist and f.title: metalyrics = lyrics.getLyrics(f.artist, f.title) else: logger.info( 'No artist/track metadata found for track: %s. Not fetching lyrics' % downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) metalyrics = None if lyrics: logger.debug( 'Adding lyrics to: %s' % downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) f.lyrics = metalyrics f.save()
def embedLyrics(downloaded_track_list): logger.info('Adding lyrics') # TODO: If adding lyrics for flac & lossy, only fetch the lyrics once # and apply it to both files for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except: logger.error('Could not read %s. Not checking lyrics' % downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) continue if f.albumartist and f.title: metalyrics = lyrics.getLyrics(f.albumartist, f.title) elif f.artist and f.title: metalyrics = lyrics.getLyrics(f.artist, f.title) else: logger.info('No artist/track metadata found for track: %s. Not fetching lyrics' % downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) metalyrics = None if lyrics: logger.debug('Adding lyrics to: %s' % downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) f.lyrics = metalyrics f.save()
def write(self): """Writes the item's metadata to the associated file. """ f = MediaFile(syspath(self.path)) for key in ITEM_KEYS_WRITABLE: if getattr(self, key): #make sure it has a value before we set it and create blank tags with wrong types setattr(f, key, getattr(self, key)) f.save()
def write(self): """Writes the item's metadata to the associated file. """ f = MediaFile(syspath(self.path)) plugins.send("write", item=self, mf=f) for key in ITEM_KEYS_WRITABLE: setattr(f, key, getattr(self, key)) f.save() # The file has a new mtime. self.mtime = self.current_mtime()
def write(self): """Writes the item's metadata to the associated file. """ f = MediaFile(syspath(self.path)) plugins.send('write', item=self, mf=f) for key in ITEM_KEYS_WRITABLE: setattr(f, key, getattr(self, key)) f.save() # The file has a new mtime. self.mtime = self.current_mtime()
def embedAlbumArt(artwork, downloaded_track_list): logger.info("Embedding album art") for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except: logger.error("Could not read %s. Not adding album art" % downloaded_track) logger.debug("Adding album art to: %s" % downloaded_track) f.art = artwork f.save()
def embedAlbumArt(artwork, downloaded_track_list): logger.info('Embedding album art') for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except: logger.error('Could not read %s. Not adding album art' % downloaded_track) continue logger.debug('Adding album art to: %s' % downloaded_track) f.art = artwork f.save()
def embedAlbumArt(artwork, downloaded_track_list): logger.info('Embedding album art') for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except: logger.error(u'Could not read %s. Not adding album art' % downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) continue logger.debug('Adding album art to: %s' % downloaded_track) f.art = artwork f.save()
def updateHave(albumpath): results = [] for r,d,f in os.walk(albumpath): for files in f: if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): results.append(os.path.join(r, files)) if results: myDB = db.DBConnection() for song in results: try: f = MediaFile(song) #logger.debug('Reading: %s' % song.decode('UTF-8')) except: logger.warn('Could not read file: %s' % song) continue else: if f.albumartist: artist = f.albumartist elif f.artist: artist = f.artist else: continue myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [unicode(song, headphones.SYS_ENCODING, errors="replace"), f.bitrate, f.format, artist, f.album, f.title])
def renameFiles(albumpath, downloaded_track_list, release): logger.info('Renaming files') try: year = release['ReleaseDate'][:4] except TypeError: year = '' # Until tagging works better I'm going to rely on the already provided metadata for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except: logger.info("MediaFile couldn't parse: " + downloaded_track) continue if not f.track: tracknumber = '' else: tracknumber = '%02d' % f.track if not f.title: basename = unicode(os.path.basename(downloaded_track), headphones.SYS_ENCODING, errors='replace') title = os.path.splitext(basename)[0] ext = os.path.splitext(basename)[1] new_file_name = helpers.cleanTitle(title) + ext else: title = f.title values = { 'tracknumber': tracknumber, 'title': title, 'artist': release['ArtistName'], 'album': release['AlbumTitle'], 'year': year } ext = os.path.splitext(downloaded_track)[1] new_file_name = helpers.replace_all(headphones.FILE_FORMAT, values).replace('/', '_') + ext new_file_name = new_file_name.replace('?', '_').replace( ':', '_').encode(headphones.SYS_ENCODING) new_file = os.path.join(albumpath, new_file_name) logger.debug('Renaming %s ---> %s' % (downloaded_track, new_file_name)) try: os.rename(downloaded_track, new_file) except Exception, e: logger.error('Error renaming file: %s. Error: %s' % (downloaded_track, e)) continue
def embedLyrics(downloaded_track_list): logger.info('Adding lyrics') for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except: logger.error('Could not read %s. Not checking lyrics' % downloaded_track) if f.albumartist and f.title: metalyrics = lyrics.getLyrics(f.albumartist, f.title) elif f.artist and f.title: metalyrics = lyrics.getLyrics(f.artist, f.title) else: logger.info('No artist/track metadata found for track: %s. Not fetching lyrics' % downloaded_track) metalyrics = None if lyrics: logger.debug('Adding lyrics to: %s' % downloaded_track) f.lyrics = metalyrics f.save()
def read(self, read_path=None): """Read the metadata from the associated file. If read_path is specified, read metadata from that file instead. """ if read_path is None: read_path = self.path else: read_path = normpath(read_path) f = MediaFile(syspath(read_path)) for key in ITEM_KEYS_META: setattr(self, key, getattr(f, key)) self.path = read_path
def updateFormat(): myDB = db.DBConnection() tracks = myDB.select('SELECT * from tracks WHERE Location IS NOT NULL and Format IS NULL') if len(tracks) > 0: logger.info('Finding media format for %s files' % len(tracks)) for track in tracks: try: f = MediaFile(track['Location']) except Exception, e: logger.info("Exception from MediaFile for: " + track['Location'] + " : " + str(e)) continue controlValueDict = {"TrackID": track['TrackID']} newValueDict = {"Format": f.format} myDB.upsert("tracks", newValueDict, controlValueDict) logger.info('Finished finding media format for %s files' % len(tracks))
def read(self, read_path=None): """Read the metadata from the associated file. If read_path is specified, read metadata from that file instead. """ if read_path is None: read_path = self.path else: read_path = normpath(read_path) f = MediaFile(syspath(read_path)) for key in ITEM_KEYS_META: setattr(self, key, getattr(f, key)) self.path = read_path # Database's mtime should now reflect the on-disk value. if read_path == self.path: self.mtime = self.current_mtime()
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None): if not dir: dir = headphones.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING)) return myDB = db.DBConnection() if not append: # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) myDB.action('DELETE from have') logger.info('Scanning music directory: %s' % dir) new_artists = [] bitrates = [] song_list = [] for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) song_dict = { 'TrackID' : f.mb_trackid, 'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre' : f.genre, 'Date' : f.date, 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, 'Location' : unicode_song_path } song_list.append(song_dict) # Now we start track matching total_number_of_songs = len(song_list) logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 for song in song_list: song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") # If the track has a trackid & releaseid (beets: albumid) that the most surefire way # of identifying a track to a specific release so we'll use that first if song['TrackID'] and song['ReleaseID']: # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table if not track: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() if track: # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo controlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['ReleaseID'] } # Insert it into the Headphones hybrid release (ReleaseID == AlbumID) hybridControlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['AlbumID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) myDB.upsert("alltracks", newValueDict, hybridControlValueDict) myDB.upsert("tracks", newValueDict, hybridControlValueDict) # Matched. Move on to the next one: continue # If we can't find it with TrackID & ReleaseID, next most specific will be # releaseid + tracktitle, although perhaps less reliable due to a higher # likelihood of variations in the song title (e.g. feat. artists) if song['ReleaseID'] and song['TrackTitle']: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() if not track: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() if track: # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table) # So use both TrackID & ReleaseID as the control values controlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['ReleaseID'] } hybridControlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['AlbumID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } # Update both tables here as well myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) myDB.upsert("alltracks", newValueDict, hybridControlValueDict) myDB.upsert("tracks", newValueDict, hybridControlValueDict) # Done continue # Next most specific will be the opposite: a TrackID and an AlbumTitle # TrackIDs span multiple releases so if something is on an official album # and a compilation, for example, this will match it to the right one # However - there may be multiple matches here if song['TrackID'] and song['AlbumTitle']: # Even though there might be multiple matches, we just need to grab one to confirm a match track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() if not track: track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() if track: # Don't need the hybridControlValueDict here since ReleaseID is not unique controlValueDict = { 'TrackID' : track['TrackID'], 'AlbumTitle' : track['AlbumTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably # even more unreliable than the previous queries, and might span multiple releases) if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if not track: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc) # This is more reliable than the former but requires some string manipulation so we'll do it only # if we can't find a match with the original data if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone() if not track: track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone() if track: controlValueDict = { 'CleanName' : track['CleanName'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track # with more specific information exists in the library, it'll overwrite these values if song['TrackID']: track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone() if not track: track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone() if track: controlValueDict = { 'TrackID' : track['TrackID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release if song['ArtistName']: new_artists.append(song['ArtistName']) else: continue # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) else: continue myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir) if not append: # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info('Updating current artist track counts') for artist in current_artists: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT into newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
def libraryScan(dir=None): if not dir: dir = headphones.MUSIC_DIR try: dir = str(dir) except UnicodeEncodeError: dir = unicode(dir).encode('unicode_escape') if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir) return myDB = db.DBConnection() # Clean up bad filepaths tracks = myDB.select( 'SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode( headphones.SYS_ENCODING)): myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) logger.info('Scanning music directory: %s' % dir) new_artists = [] bitrates = [] myDB.action('DELETE from have') for r, d, f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors='replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + file) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Try to find a match based on artist/album/tracktitle if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: continue if f_artist and f.album and f.title: track = myDB.action( 'SELECT TrackID from tracks WHERE CleanName LIKE ?', [ helpers.cleanName(f_artist + ' ' + f.album + ' ' + f.title) ]).fetchone() if not track: track = myDB.action( 'SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() if track: myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']]) continue # Try to match on mbid if available and we couldn't find a match based on metadata if f.mb_trackid: # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action( 'SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() if track: myDB.action( 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']]) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release new_artists.append(f_artist) # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database myDB.action( 'INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [ f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist + ' ' + f.album + ' ' + f.title), f.format ]) logger.info('Completed scanning of directory: %s' % dir) logger.info('Checking filepaths to see if we can find any matches') # Now check empty file paths to see if we can find a match based on their folder format tracks = myDB.select('SELECT * from tracks WHERE Location IS NULL') for track in tracks: release = myDB.action('SELECT * from albums WHERE AlbumID=?', [track['AlbumID']]).fetchone() try: year = release['ReleaseDate'][:4] except TypeError: year = '' artist = release['ArtistName'].replace('/', '_') album = release['AlbumTitle'].replace('/', '_') releasetype = release['Type'].replace('/', '_') if release['ArtistName'].startswith('The '): sortname = release['ArtistName'][4:] else: sortname = release['ArtistName'] if sortname.isdigit(): firstchar = '0-9' else: firstchar = sortname[0] albumvalues = { '$Artist': artist, '$Album': album, '$Year': year, '$Type': releasetype, '$First': firstchar, '$artist': artist.lower(), '$album': album.lower(), '$year': year, '$type': releasetype.lower(), '$first': firstchar.lower() } folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues) folder = folder.replace('./', '_/').replace(':', '_').replace('?', '_') if folder.endswith('.'): folder = folder.replace(folder[len(folder) - 1], '_') if not track['TrackNumber']: tracknumber = '' else: tracknumber = '%02d' % track['TrackNumber'] title = track['TrackTitle'] trackvalues = { '$Track': tracknumber, '$Title': title, '$Artist': release['ArtistName'], '$Album': release['AlbumTitle'], '$Year': year, '$track': tracknumber, '$title': title.lower(), '$artist': release['ArtistName'].lower(), '$album': release['AlbumTitle'].lower(), '$year': year } new_file_name = helpers.replace_all( headphones.FILE_FORMAT, trackvalues).replace('/', '_') + '.*' new_file_name = new_file_name.replace('?', '_').replace(':', '_') full_path_to_file = os.path.normpath( os.path.join(headphones.MUSIC_DIR, folder, new_file_name)).encode(headphones.SYS_ENCODING, 'replace') match = glob.glob(full_path_to_file) if match: logger.info('Found a match: %s. Writing MBID to metadata' % match[0]) unipath = unicode(match[0], headphones.SYS_ENCODING, errors='replace') myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [unipath, track['TrackID']]) myDB.action('DELETE from have WHERE Location=?', [unipath]) # Try to insert the appropriate track id so we don't have to keep doing this try: f = MediaFile(match[0]) f.mb_trackid = track['TrackID'] f.save() myDB.action( 'UPDATE tracks SET BitRate=?, Format=? WHERE TrackID=?', [f.bitrate, f.format, track['TrackID']]) logger.debug('Wrote mbid to track: %s' % match[0]) except: logger.error('Error embedding track id into: %s' % match[0]) continue logger.info('Done checking empty filepaths') logger.info('Done syncing library with directory: %s' % dir) # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [ f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists] ] # Update track counts logger.info('Updating track counts') for artist in current_artists: havetracks = len( myDB.select( 'SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len( myDB.select( 'SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info( 'To add these artists, go to Manage->Manage New Artists') headphones.NEW_ARTISTS = artist_list if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000
#reload downloaded_track_list = [] for r, d, f in os.walk(albumpath): for file in f: if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): downloaded_track_list.append(os.path.join(r, file)) # test #1: metadata - usually works logger.debug('Verifying metadata...') for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except Exception, e: logger.info( u"Exception from MediaFile for: " + downloaded_track.decode(headphones.SYS_ENCODING, 'replace') + u" : " + unicode(e)) continue if not f.artist: continue if not f.album: continue metaartist = helpers.latinToAscii(f.artist.lower()).encode('UTF-8') dbartist = helpers.latinToAscii( release['ArtistName'].lower()).encode('UTF-8')
def libraryScan(dir=None): if not dir: dir = headphones.MUSIC_DIR try: dir = str(dir) except UnicodeEncodeError: dir = unicode(dir).encode("unicode_escape") if not os.path.isdir(dir): logger.warn("Cannot find directory: %s. Not scanning" % dir) return myDB = db.DBConnection() # Clean up bad filepaths tracks = myDB.select("SELECT Location, TrackID from tracks WHERE Location IS NOT NULL") for track in tracks: if not os.path.isfile(track["Location"].encode(headphones.SYS_ENCODING)): myDB.action( "UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?", [None, None, None, track["TrackID"]], ) logger.info("Scanning music directory: %s" % dir) new_artists = [] bitrates = [] myDB.action("DELETE from have") for r, d, f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith("." + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors="replace") # Try to read the metadata try: f = MediaFile(song) except: logger.error("Cannot read file: " + file) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Try to find a match based on artist/album/tracktitle if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: continue if f_artist and f.album and f.title: track = myDB.action( "SELECT TrackID from tracks WHERE CleanName LIKE ?", [helpers.cleanName(f_artist + " " + f.album + " " + f.title)], ).fetchone() if not track: track = myDB.action( "SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?", [f_artist, f.album, f.title], ).fetchone() if track: myDB.action( "UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?", [file, f.bitrate, f.format, track["TrackID"]], ) continue # Try to match on mbid if available and we couldn't find a match based on metadata if f.mb_trackid: # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action("SELECT TrackID from tracks WHERE TrackID=?", [f.mb_trackid]).fetchone() if track: myDB.action( "UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?", [file, f.bitrate, f.format, track["TrackID"]], ) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release new_artists.append(f_artist) # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database myDB.action( "INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [ f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist + " " + f.album + " " + f.title), f.format, ], ) logger.info("Completed scanning of directory: %s" % dir) logger.info("Checking filepaths to see if we can find any matches") # Now check empty file paths to see if we can find a match based on their folder format tracks = myDB.select("SELECT * from tracks WHERE Location IS NULL") for track in tracks: release = myDB.action("SELECT * from albums WHERE AlbumID=?", [track["AlbumID"]]).fetchone() try: year = release["ReleaseDate"][:4] except TypeError: year = "" artist = release["ArtistName"].replace("/", "_") album = release["AlbumTitle"].replace("/", "_") releasetype = release["Type"].replace("/", "_") if release["ArtistName"].startswith("The "): sortname = release["ArtistName"][4:] else: sortname = release["ArtistName"] if sortname.isdigit(): firstchar = "0-9" else: firstchar = sortname[0] albumvalues = { "$Artist": artist, "$Album": album, "$Year": year, "$Type": releasetype, "$First": firstchar, "$artist": artist.lower(), "$album": album.lower(), "$year": year, "$type": releasetype.lower(), "$first": firstchar.lower(), } folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues) folder = folder.replace("./", "_/").replace(":", "_").replace("?", "_") if folder.endswith("."): folder = folder.replace(folder[len(folder) - 1], "_") if not track["TrackNumber"]: tracknumber = "" else: tracknumber = "%02d" % track["TrackNumber"] title = track["TrackTitle"] trackvalues = { "$Track": tracknumber, "$Title": title, "$Artist": release["ArtistName"], "$Album": release["AlbumTitle"], "$Year": year, "$track": tracknumber, "$title": title.lower(), "$artist": release["ArtistName"].lower(), "$album": release["AlbumTitle"].lower(), "$year": year, } new_file_name = helpers.replace_all(headphones.FILE_FORMAT, trackvalues).replace("/", "_") + ".*" new_file_name = new_file_name.replace("?", "_").replace(":", "_") full_path_to_file = os.path.normpath(os.path.join(headphones.MUSIC_DIR, folder, new_file_name)).encode( headphones.SYS_ENCODING, "replace" ) match = glob.glob(full_path_to_file) if match: logger.info("Found a match: %s. Writing MBID to metadata" % match[0]) unipath = unicode(match[0], headphones.SYS_ENCODING, errors="replace") myDB.action("UPDATE tracks SET Location=? WHERE TrackID=?", [unipath, track["TrackID"]]) myDB.action("DELETE from have WHERE Location=?", [unipath]) # Try to insert the appropriate track id so we don't have to keep doing this try: f = MediaFile(match[0]) f.mb_trackid = track["TrackID"] f.save() myDB.action( "UPDATE tracks SET BitRate=?, Format=? WHERE TrackID=?", [f.bitrate, f.format, track["TrackID"]] ) logger.debug("Wrote mbid to track: %s" % match[0]) except: logger.error("Error embedding track id into: %s" % match[0]) continue logger.info("Done checking empty filepaths") logger.info("Done syncing library with directory: %s" % dir) # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select("SELECT ArtistName, ArtistID from artists") artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info("Updating track counts") for artist in current_artists: havetracks = len( myDB.select( "SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL", [artist["ArtistID"]] ) ) + len(myDB.select("SELECT TrackTitle from have WHERE ArtistName like ?", [artist["ArtistName"]])) myDB.action("UPDATE artists SET HaveTracks=? WHERE ArtistID=?", [havetracks, artist["ArtistID"]]) logger.info("Found %i new artists" % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info("Importing %i new artists" % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info("To add these artists, go to Manage->Manage New Artists") headphones.NEW_ARTISTS = artist_list if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None): if not dir: dir = headphones.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace')) return myDB = db.DBConnection() if not append: # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) myDB.action('DELETE from have') logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) new_artists = [] bitrates = [] song_list = [] for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) song_dict = { 'TrackID' : f.mb_trackid, 'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre' : f.genre, 'Date' : f.date, 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, 'Location' : unicode_song_path } song_list.append(song_dict) # Now we start track matching total_number_of_songs = len(song_list) logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 for song in song_list: song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") # If the track has a trackid & releaseid (beets: albumid) that the most surefire way # of identifying a track to a specific release so we'll use that first if song['TrackID'] and song['ReleaseID']: # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table if not track: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() if track: # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo controlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['ReleaseID'] } # Insert it into the Headphones hybrid release (ReleaseID == AlbumID) hybridControlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['AlbumID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) myDB.upsert("alltracks", newValueDict, hybridControlValueDict) myDB.upsert("tracks", newValueDict, hybridControlValueDict) # Matched. Move on to the next one: continue # If we can't find it with TrackID & ReleaseID, next most specific will be # releaseid + tracktitle, although perhaps less reliable due to a higher # likelihood of variations in the song title (e.g. feat. artists) if song['ReleaseID'] and song['TrackTitle']: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() if not track: track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() if track: # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table) # So use both TrackID & ReleaseID as the control values controlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['ReleaseID'] } hybridControlValueDict = { 'TrackID' : track['TrackID'], 'ReleaseID' : track['AlbumID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } # Update both tables here as well myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) myDB.upsert("alltracks", newValueDict, hybridControlValueDict) myDB.upsert("tracks", newValueDict, hybridControlValueDict) # Done continue # Next most specific will be the opposite: a TrackID and an AlbumTitle # TrackIDs span multiple releases so if something is on an official album # and a compilation, for example, this will match it to the right one # However - there may be multiple matches here if song['TrackID'] and song['AlbumTitle']: # Even though there might be multiple matches, we just need to grab one to confirm a match track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() if not track: track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() if track: # Don't need the hybridControlValueDict here since ReleaseID is not unique controlValueDict = { 'TrackID' : track['TrackID'], 'AlbumTitle' : track['AlbumTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably # even more unreliable than the previous queries, and might span multiple releases) if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if not track: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc) # This is more reliable than the former but requires some string manipulation so we'll do it only # if we can't find a match with the original data if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone() if not track: track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone() if track: controlValueDict = { 'CleanName' : track['CleanName'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track # with more specific information exists in the library, it'll overwrite these values if song['TrackID']: track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone() if not track: track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone() if track: controlValueDict = { 'TrackID' : track['TrackID'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release if song['ArtistName']: new_artists.append(song['ArtistName']) else: continue # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) else: continue myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info('Updating current artist track counts') for artist in current_artists: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT into newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
for track in tracks: try: f = MediaFile(track['Location']) except Exception, e: logger.info("Exception from MediaFile for: " + track['Location'] + " : " + str(e)) continue controlValueDict = {"TrackID": track['TrackID']} newValueDict = {"Format": f.format} myDB.upsert("tracks", newValueDict, controlValueDict) logger.info('Finished finding media format for %s files' % len(tracks)) havetracks = myDB.select('SELECT * from have WHERE Location IS NOT NULL and Format IS NULL') if len(havetracks) > 0: logger.info('Finding media format for %s files' % len(havetracks)) for track in havetracks: try: f = MediaFile(track['Location']) except Exception, e: logger.info("Exception from MediaFile for: " + track['Location'] + " : " + str(e)) continue controlValueDict = {"TrackID": track['TrackID']} newValueDict = {"Format": f.format} myDB.upsert("have", newValueDict, controlValueDict) logger.info('Finished finding media format for %s files' % len(havetracks)) def getHybridRelease(fullreleaselist): """ Returns a dictionary of best group of tracks from the list of releases & earliest release date """ if len(fullreleaselist) == 0: raise Exception("getHybridRelease was called with an empty fullreleaselist") sortable_release_list = []
def libraryScan(dir=None): if not dir: dir = headphones.MUSIC_DIR try: dir = str(dir) except UnicodeEncodeError: dir = unicode(dir).encode('unicode_escape') if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir) return myDB = db.DBConnection() # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) logger.info('Scanning music directory: %s' % dir) new_artists = [] bitrates = [] myDB.action('DELETE from have') for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors='replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + file) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Try to find a match based on artist/album/tracktitle if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: continue if f_artist and f.album and f.title: track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone() if not track: track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']]) continue # Try to match on mbid if available and we couldn't find a match based on metadata if f.mb_trackid: # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']]) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release new_artists.append(f_artist) # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist+' '+f.album+' '+f.title), f.format]) logger.info('Completed scanning of directory: %s' % dir) logger.info('Checking filepaths to see if we can find any matches') # Now check empty file paths to see if we can find a match based on their folder format tracks = myDB.select('SELECT * from tracks WHERE Location IS NULL') for track in tracks: release = myDB.action('SELECT * from albums WHERE AlbumID=?', [track['AlbumID']]).fetchone() try: year = release['ReleaseDate'][:4] except TypeError: year = '' artist = release['ArtistName'].replace('/', '_') album = release['AlbumTitle'].replace('/', '_') if release['ArtistName'].startswith('The '): sortname = release['ArtistName'][4:] else: sortname = release['ArtistName'] if sortname.isdigit(): firstchar = '0-9' else: firstchar = sortname[0] lowerfirst = firstchar.lower() albumvalues = { 'artist': artist, 'album': album, 'year': year, 'first': firstchar, 'lowerfirst': lowerfirst } folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues) folder = folder.replace('./', '_/').replace(':','_').replace('?','_') if folder.endswith('.'): folder = folder.replace(folder[len(folder)-1], '_') if not track['TrackNumber']: tracknumber = '' else: tracknumber = '%02d' % track['TrackNumber'] trackvalues = { 'tracknumber': tracknumber, 'title': track['TrackTitle'], 'artist': release['ArtistName'], 'album': release['AlbumTitle'], 'year': year } new_file_name = helpers.replace_all(headphones.FILE_FORMAT, trackvalues).replace('/','_') + '.*' new_file_name = new_file_name.replace('?','_').replace(':', '_') full_path_to_file = os.path.normpath(os.path.join(headphones.MUSIC_DIR, folder, new_file_name)).encode(headphones.SYS_ENCODING, 'replace') match = glob.glob(full_path_to_file) if match: logger.info('Found a match: %s. Writing MBID to metadata' % match[0]) unipath = unicode(match[0], headphones.SYS_ENCODING, errors='replace') myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [unipath, track['TrackID']]) myDB.action('DELETE from have WHERE Location=?', [unipath]) # Try to insert the appropriate track id so we don't have to keep doing this try: f = MediaFile(match[0]) f.mb_trackid = track['TrackID'] f.save() myDB.action('UPDATE tracks SET BitRate=?, Format=? WHERE TrackID=?', [f.bitrate, f.format, track['TrackID']]) logger.debug('Wrote mbid to track: %s' % match[0]) except: logger.error('Error embedding track id into: %s' % match[0]) continue logger.info('Done checking empty filepaths') logger.info('Done syncing library with directory: %s' % dir) # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info('Updating track counts') for artist in current_artists: havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') headphones.NEW_ARTISTS = artist_list if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
def fileSystemScan(self): # Now check empty file paths to see if we can find a match based on their folder format tracks = myDB.select('SELECT * from tracks WHERE Location IS NULL') for track in tracks: release = myDB.action('SELECT * from albums WHERE AlbumID=?', [track['AlbumID']]).fetchone() try: year = release['ReleaseDate'][:4] except TypeError: year = '' artist = release['ArtistName'].replace('/', '_') album = release['AlbumTitle'].replace('/', '_') if release['ArtistName'].startswith('The '): sortname = release['ArtistName'][4:] else: sortname = release['ArtistName'] if sortname.isdigit(): firstchar = '0-9' else: firstchar = sortname[0] lowerfirst = firstchar.lower() albumvalues = { 'artist': artist, 'album': album, 'year': year, 'first': firstchar, 'lowerfirst': lowerfirst } folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues) folder = folder.replace('./', '_/').replace(':','_').replace('?','_') if folder.endswith('.'): folder = folder.replace(folder[len(folder)-1], '_') if not track['TrackNumber']: tracknumber = '' else: tracknumber = '%02d' % track['TrackNumber'] trackvalues = { 'tracknumber': tracknumber, 'title': track['TrackTitle'], 'artist': release['ArtistName'], 'album': release['AlbumTitle'], 'year': year } new_file_name = helpers.replace_all(headphones.FILE_FORMAT, trackvalues).replace('/','_') + '.*' new_file_name = new_file_name.replace('?','_').replace(':', '_') full_path_to_file = os.path.normpath(os.path.join(headphones.MUSIC_DIR, folder, new_file_name)).encode(headphones.SYS_ENCODING, 'replace') match = glob.glob(full_path_to_file) if match: logger.info('Found a match: %s. Writing MBID to metadata' % match[0]) unipath = unicode(match[0], headphones.SYS_ENCODING, errors='replace') myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [unipath, track['TrackID']]) myDB.action('DELETE from have WHERE Location=?', [unipath]) # Try to insert the appropriate track id so we don't have to keep doing this try: f = MediaFile(match[0]) f.mb_trackid = track['TrackID'] f.save() myDB.action('UPDATE tracks SET BitRate=? WHERE TrackID=?', [f.bitrate, track['TrackID']]) logger.debug('Wrote mbid to track: %s' % match[0]) except: logger.error('Error embedding track id into: %s' % match[0]) continue
def renameFiles(albumpath, downloaded_track_list, release): logger.info('Renaming files') try: year = release['ReleaseDate'][:4] except TypeError: year = '' # Until tagging works better I'm going to rely on the already provided metadata for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) except: logger.info( "MediaFile couldn't parse: " + downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) continue if not f.disc: discnumber = '' else: discnumber = '%d' % f.disc if not f.track: tracknumber = '' else: tracknumber = '%02d' % f.track if not f.title: basename = os.path.basename( downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) title = os.path.splitext(basename)[0] ext = os.path.splitext(basename)[1] new_file_name = helpers.cleanTitle(title) + ext else: title = f.title if release['ArtistName'] == "Various Artists" and f.artist: artistname = f.artist else: artistname = release['ArtistName'] if artistname.startswith('The '): sortname = artistname[4:] + ", The" else: sortname = artistname values = { '$Disc': discnumber, '$Track': tracknumber, '$Title': title, '$Artist': artistname, '$SortArtist': sortname, '$Album': release['AlbumTitle'], '$Year': year, '$disc': discnumber, '$track': tracknumber, '$title': title.lower(), '$artist': artistname.lower(), '$sortartist': sortname.lower(), '$album': release['AlbumTitle'].lower(), '$year': year } ext = os.path.splitext(downloaded_track)[1] new_file_name = helpers.replace_all(headphones.FILE_FORMAT.strip(), values).replace('/', '_') + ext new_file_name = new_file_name.replace('?', '_').replace( ':', '_').encode(headphones.SYS_ENCODING, 'replace') if new_file_name.startswith('.'): new_file_name = new_file_name.replace(0, '_') new_file = os.path.join(albumpath, new_file_name) if downloaded_track == new_file_name: logger.debug( "Renaming for: " + downloaded_track.decode(headphones.SYS_ENCODING, 'replace') + " is not neccessary") continue logger.debug( 'Renaming %s ---> %s' % (downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), new_file_name.decode(headphones.SYS_ENCODING, 'replace'))) try: os.rename(downloaded_track, new_file) except Exception, e: logger.error('Error renaming file: %s. Error: %s' % (downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), e)) continue
def encode(albumPath): # Return if xld details not found if XLD: global xldProfile (xldProfile, xldFormat, xldBitrate) = getXldProfile.getXldProfile(headphones.XLDPROFILE) if not xldFormat: logger.error( u'Details for xld profile "%s" not found, will not be reencoded' % (xldProfile)) return None tempDirEncode = os.path.join(albumPath, "temp") musicFiles = [] musicFinalFiles = [] musicTempFiles = [] encoder = "" startAlbumTime = time.time() ifencoded = 0 if not os.path.exists(tempDirEncode): os.mkdir(tempDirEncode) else: shutil.rmtree(tempDirEncode) time.sleep(1) os.mkdir(tempDirEncode) for r, d, f in os.walk(albumPath): for music in f: if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): if not XLD: encoderFormat = headphones.ENCODEROUTPUTFORMAT.encode( headphones.SYS_ENCODING) else: xldMusicFile = os.path.join(r, music) xldInfoMusic = MediaFile(xldMusicFile) encoderFormat = xldFormat if (headphones.ENCODERLOSSLESS): ext = os.path.normpath( os.path.splitext(music)[1].lstrip(".")).lower() if not XLD and ext == 'flac' or XLD and ( ext != xldFormat and (xldInfoMusic.bitrate / 1000 > 500)): musicFiles.append(os.path.join(r, music)) musicTemp = os.path.normpath( os.path.splitext(music)[0] + '.' + encoderFormat) musicTempFiles.append( os.path.join(tempDirEncode, musicTemp)) else: logger.debug('Music "%s" is already encoded' % (music)) else: musicFiles.append(os.path.join(r, music)) musicTemp = os.path.normpath( os.path.splitext(music)[0] + '.' + encoderFormat) musicTempFiles.append( os.path.join(tempDirEncode, musicTemp)) if XLD: if headphones.ENCODERFOLDER: encoder = os.path.join( headphones.ENCODERFOLDER.encode(headphones.SYS_ENCODING), 'xld') else: encoder = os.path.join('/Applications', 'xld') elif headphones.ENCODER == 'lame': encoder = os.path.join( headphones.ENCODERFOLDER.encode(headphones.SYS_ENCODING), 'lame') elif headphones.ENCODER == 'ffmpeg': encoder = os.path.join( headphones.ENCODERFOLDER.encode(headphones.SYS_ENCODING), 'ffmpeg') i = 0 for music in musicFiles: infoMusic = MediaFile(music) if XLD: if xldBitrate and (infoMusic.bitrate / 1000 <= xldBitrate): logger.info( 'Music "%s" has bitrate <= "%skbit", will not be reencoded' % (music.decode(headphones.SYS_ENCODING, 'replace'), xldBitrate)) else: command(encoder, music, musicTempFiles[i], albumPath) ifencoded = 1 elif headphones.ENCODER == 'lame': if not any( music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + x) for x in ["mp3", "wav"]): logger.warn( u'Lame cant encode "%s" format for "%s", use ffmpeg' % (os.path.splitext(music)[1].decode(headphones.SYS_ENCODING, 'replace'), music.decode(headphones.SYS_ENCODING, 'replace'))) else: if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.mp3') and (int(infoMusic.bitrate / 1000) <= headphones.BITRATE)): logger.info( 'Music "%s" has bitrate<="%skbit" will not be reencoded' % (music.decode(headphones.SYS_ENCODING, 'replace'), headphones.BITRATE)) else: command(encoder, music, musicTempFiles[i], albumPath) ifencoded = 1 else: if headphones.ENCODEROUTPUTFORMAT == 'ogg': if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.ogg'): logger.warn( 'Can not reencode .ogg music "%s"' % (music.decode(headphones.SYS_ENCODING, 'replace'))) else: command(encoder, music, musicTempFiles[i], albumPath) ifencoded = 1 elif (headphones.ENCODEROUTPUTFORMAT == 'mp3' or headphones.ENCODEROUTPUTFORMAT == 'm4a'): if (music.decode(headphones.SYS_ENCODING, 'replace').lower( ).endswith('.' + headphones.ENCODEROUTPUTFORMAT) and (int(infoMusic.bitrate / 1000) <= headphones.BITRATE)): logger.info( 'Music "%s" has bitrate<="%skbit" will not be reencoded' % (music.decode(headphones.SYS_ENCODING, 'replace'), headphones.BITRATE)) else: command(encoder, music, musicTempFiles[i], albumPath) ifencoded = 1 i = i + 1 shutil.rmtree(tempDirEncode) time.sleep(1) for r, d, f in os.walk(albumPath): for music in f: if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): musicFinalFiles.append(os.path.join(r, music)) if ifencoded == 0: logger.info('Encoding for folder "%s" is not needed' % (albumPath.decode(headphones.SYS_ENCODING, 'replace'))) return musicFinalFiles
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False): if cron and not headphones.LIBRARYSCAN: return if not dir: if not headphones.MUSIC_DIR: return else: dir = headphones.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append: dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace')) return myDB = db.DBConnection() new_artists = [] logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL') for track in tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) if not os.path.isfile(encoded_track_string): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have') for track in del_have_tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) if not os.path.isfile(encoded_track_string): if track['ArtistName']: #Make sure deleted files get accounted for when updating artist track counts new_artists.append(track['ArtistName']) myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) ###############myDB.action('DELETE from have') bitrates = [] song_list = [] new_song_count = 0 file_count = 0 latest_subdirectory = [] for r,d,f in os.walk(dir): #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one #using a list comprehension will not work correctly for nested subdirectories (os.walk keeps its original list) for directory in d[:]: if directory.startswith("."): d.remove(directory) for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): subdirectory = r.replace(dir,'') latest_subdirectory.append(subdirectory) if file_count == 0 and r.replace(dir,'') !='': logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) elif latest_subdirectory[file_count] != latest_subdirectory[file_count-1] and file_count !=0: logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: f_artist = None # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) if f_artist and f.album and f.title: CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title) else: CleanName = None controlValueDict = {'Location' : unicode_song_path} newValueDict = { 'TrackID' : f.mb_trackid, #'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, 'TrackLength': f.length, 'Genre' : f.genre, 'Date' : f.date, 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, 'CleanName' : CleanName } #song_list.append(song_dict) check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. if not check_exist_song: #This is a new track if f_artist: new_artists.append(f_artist) myDB.upsert("have", newValueDict, controlValueDict) new_song_count+=1 else: if check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: #Important track metadata has been modified, need to run matcher again if f_artist and f_artist != check_exist_song['ArtistName']: new_artists.append(f_artist) elif f_artist and f_artist == check_exist_song['ArtistName'] and check_exist_song['Matched'] != "Ignored": new_artists.append(f_artist) else: continue newValueDict['Matched'] = None myDB.upsert("have", newValueDict, controlValueDict) myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) new_song_count+=1 else: #This track information hasn't changed if f_artist and check_exist_song['Matched'] != "Ignored": new_artists.append(f_artist) file_count+=1 # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]) total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0] logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 latest_artist = [] for song in song_list: latest_artist.append(song['ArtistName']) if song_count == 0: logger.info("Now matching songs by %s" % song['ArtistName']) elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0: logger.info("Now matching songs by %s" % song['ArtistName']) #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle'] song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if track: controlValueDict = { 'CleanName' : track['CleanName']} newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if alltrack: controlValueDict = { 'ArtistName' : alltrack['ArtistName'], 'AlbumTitle' : alltrack['AlbumTitle'], 'TrackTitle' : alltrack['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if alltrack: controlValueDict = { 'CleanName' : alltrack['CleanName']} newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) else: controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: logger.info('Updating scanned artist track counts') # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') #There was a bug where artists with special characters (-,') would show up in new artists. artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]] artists_checked = [f for f in unique_artists if helpers.cleanName(f).lower() in [helpers.cleanName(x[0]).lower() for x in current_artists]] # Update track counts for artist in artists_checked: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist])) #Note, some people complain about having "artist have tracks" > # of tracks total in artist official releases # (can fix by getting rid of second len statement) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') #myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 else: # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) update_album_status() logger.info('Library scan complete')
def encode(albumPath): # Return if xld details not found if XLD: global xldProfile (xldProfile, xldFormat, xldBitrate) = getXldProfile.getXldProfile(headphones.XLDPROFILE) if not xldFormat: logger.error( u'Details for xld profile %s not found, files will not be re-encoded' % (xldProfile)) return None tempDirEncode = os.path.join(albumPath, "temp") musicFiles = [] musicFinalFiles = [] musicTempFiles = [] encoder = "" if not os.path.exists(tempDirEncode): os.mkdir(tempDirEncode) else: shutil.rmtree(tempDirEncode) time.sleep(1) os.mkdir(tempDirEncode) for r, d, f in os.walk(albumPath): for music in f: if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): if not XLD: encoderFormat = headphones.ENCODEROUTPUTFORMAT.encode( headphones.SYS_ENCODING) else: xldMusicFile = os.path.join(r, music) xldInfoMusic = MediaFile(xldMusicFile) encoderFormat = xldFormat if (headphones.ENCODERLOSSLESS): ext = os.path.normpath( os.path.splitext(music)[1].lstrip(".")).lower() if not XLD and ext == 'flac' or XLD and ( ext != xldFormat and (xldInfoMusic.bitrate / 1000 > 400)): musicFiles.append(os.path.join(r, music)) musicTemp = os.path.normpath( os.path.splitext(music)[0] + '.' + encoderFormat) musicTempFiles.append( os.path.join(tempDirEncode, musicTemp)) else: logger.debug('%s is already encoded' % (music)) else: musicFiles.append(os.path.join(r, music)) musicTemp = os.path.normpath( os.path.splitext(music)[0] + '.' + encoderFormat) musicTempFiles.append( os.path.join(tempDirEncode, musicTemp)) if headphones.ENCODER_PATH: encoder = headphones.ENCODER_PATH.encode(headphones.SYS_ENCODING) else: if XLD: encoder = os.path.join('/Applications', 'xld') elif headphones.ENCODER == 'lame': if headphones.SYS_PLATFORM == "win32": ## NEED THE DEFAULT LAME INSTALL ON WIN! encoder = "C:/Program Files/lame/lame.exe" else: encoder = "lame" elif headphones.ENCODER == 'ffmpeg': if headphones.SYS_PLATFORM == "win32": encoder = "C:/Program Files/ffmpeg/bin/ffmpeg.exe" else: encoder = "ffmpeg" i = 0 encoder_failed = False for music in musicFiles: infoMusic = MediaFile(music) encode = False if XLD: if xldBitrate and (infoMusic.bitrate / 1000 <= xldBitrate): logger.info('%s has bitrate <= %skb, will not be re-encoded' % (music.decode(headphones.SYS_ENCODING, 'replace'), xldBitrate)) else: encode = True elif headphones.ENCODER == 'lame': if not any( music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + x) for x in ["mp3", "wav"]): logger.warn( u'Lame cannot encode %s format for %s, use ffmpeg' % (os.path.splitext(music)[1].decode(headphones.SYS_ENCODING, 'replace'), music.decode(headphones.SYS_ENCODING, 'replace'))) else: if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.mp3') and (int(infoMusic.bitrate / 1000) <= headphones.BITRATE)): logger.info( '%s has bitrate <= %skb, will not be re-encoded' % (music.decode(headphones.SYS_ENCODING, 'replace'), headphones.BITRATE)) else: encode = True else: if headphones.ENCODEROUTPUTFORMAT == 'ogg': if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.ogg'): logger.warn( 'Cannot re-encode .ogg %s' % (music.decode(headphones.SYS_ENCODING, 'replace'))) else: encode = True elif (headphones.ENCODEROUTPUTFORMAT == 'mp3' or headphones.ENCODEROUTPUTFORMAT == 'm4a'): if (music.decode(headphones.SYS_ENCODING, 'replace').lower( ).endswith('.' + headphones.ENCODEROUTPUTFORMAT) and (int(infoMusic.bitrate / 1000) <= headphones.BITRATE)): logger.info( '%s has bitrate <= %skb, will not be re-encoded' % (music.decode(headphones.SYS_ENCODING, 'replace'), headphones.BITRATE)) else: encode = True # encode if encode: if not command(encoder, music, musicTempFiles[i], albumPath): encoder_failed = True break else: musicFiles[i] = None musicTempFiles[i] = None i = i + 1 musicFiles = filter(None, musicFiles) musicTempFiles = filter(None, musicTempFiles) # check all files to be encoded now exist in temp directory if not encoder_failed and musicTempFiles: for dest in musicTempFiles: if not os.path.exists(dest): encoder_failed = True logger.error( 'Encoded file %s does not exist in the destination temp directory' % (dest.decode(headphones.SYS_ENCODING, 'replace'))) # No errors, move from temp to parent if not encoder_failed and musicTempFiles: i = 0 for dest in musicTempFiles: if os.path.exists(dest): source = musicFiles[i] if headphones.DELETE_LOSSLESS_FILES: os.remove(source) check_dest = os.path.join(albumPath, os.path.split(dest)[1]) if os.path.exists(check_dest): os.remove(check_dest) try: shutil.move(dest, albumPath) except Exception, e: logger.error( 'Could not move %s to %s : %s' % (dest.decode(headphones.SYS_ENCODING, 'replace'), albumPath.decode(headphones.SYS_ENCODING, 'replace'), e)) encoder_failed = True break i += 1
def encode(albumPath): tempDirEncode = os.path.join(albumPath, "temp") musicFiles = [] musicFinalFiles = [] musicTempFiles = [] encoder = "" startAlbumTime = time.clock() ifencoded = 0 if not os.path.exists(tempDirEncode): os.mkdir(tempDirEncode) else: shutil.rmtree(tempDirEncode) time.sleep(1) os.mkdir(tempDirEncode) for r, d, f in os.walk(albumPath): for music in f: if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): if (headphones.ENCODERLOSSLESS): if (music.lower().endswith('.flac')): musicFiles.append(os.path.join(r, music)) musicTemp = os.path.normpath( os.path.splitext(music)[0] + '.' + headphones.ENCODEROUTPUTFORMAT).encode( headphones.SYS_ENCODING) musicTempFiles.append( os.path.join(tempDirEncode, musicTemp)) else: logger.warn('Music "%s" is already encoded' % (music)) else: musicFiles.append(os.path.join(r, music)) musicTemp = os.path.normpath( os.path.splitext(music)[0] + '.' + headphones.ENCODEROUTPUTFORMAT).encode( headphones.SYS_ENCODING) musicTempFiles.append( os.path.join(tempDirEncode, musicTemp)) if headphones.ENCODER == 'lame': encoder = os.path.join(headphones.ENCODERFOLDER, 'lame') elif headphones.ENCODER == 'ffmpeg': encoder = os.path.join(headphones.ENCODERFOLDER, 'ffmpeg') i = 0 for music in musicFiles: infoMusic = MediaFile(music) if headphones.ENCODER == 'lame': if not any(music.lower().endswith('.' + x) for x in ["mp3", "wav"]): logger.warn( 'Lame cant encode "%s" format for "%s", use ffmpeg' % (os.path.splitext(music)[1], music)) else: if (music.lower().endswith('.mp3') and (infoMusic.bitrate / 1000 <= headphones.BITRATE)): logger.warn( 'Music "%s" has bitrate<="%skbit" will not be reencoded' % (music, headphones.BITRATE)) else: command(encoder, music, musicTempFiles[i], albumPath) ifencoded = 1 else: if headphones.ENCODEROUTPUTFORMAT == 'ogg': if music.lower().endswith('.ogg'): logger.warn('Can not reencode .ogg music "%s"' % (music)) else: command(encoder, music, musicTempFiles[i], albumPath) ifencoded = 1 elif (headphones.ENCODEROUTPUTFORMAT == 'mp3' or headphones.ENCODEROUTPUTFORMAT == 'm4a'): if (music.lower().endswith('.' + headphones.ENCODEROUTPUTFORMAT) and (infoMusic.bitrate / 1000 <= headphones.BITRATE)): logger.warn( 'Music "%s" has bitrate<="%skbit" will not be reencoded' % (music, headphones.BITRATE)) else: command(encoder, music, musicTempFiles[i], albumPath) ifencoded = 1 i = i + 1 shutil.rmtree(tempDirEncode) time.sleep(1) for r, d, f in os.walk(albumPath): for music in f: if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): musicFinalFiles.append(os.path.join(r, music)) if ifencoded == 0: logger.info('Encoding for folder "%s" is not needed' % (albumPath)) return musicFinalFiles
def libraryScan(dir=None): if not dir: dir = headphones.MUSIC_DIR dir = dir.encode(headphones.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING)) return myDB = db.DBConnection() # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') for track in tracks: if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) logger.info('Scanning music directory: %s' % dir) new_artists = [] bitrates = [] myDB.action('DELETE from have') for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace') # Try to read the metadata try: f = MediaFile(song) except: logger.error('Cannot read file: ' + unicode_song_path) continue # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) # Try to find a match based on artist/album/tracktitle if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: continue if f_artist and f.album and f.title: track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone() if not track: track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']]) continue # Try to match on mbid if available and we couldn't find a match based on metadata if f.mb_trackid: # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() if track: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']]) continue # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release new_artists.append(f_artist) # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, unicode_song_path, helpers.cleanName(f_artist+' '+f.album+' '+f.title), f.format]) logger.info('Completed scanning directory: %s' % dir) # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts logger.info('Updating track counts') for artist in current_artists: havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): if headphones.ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') myDB.action('DELETE from newartists') for artist in artist_list: myDB.action('INSERT into newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000