Exemple #1
0
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None):

    if not dir:
        dir = headphones.MUSIC_DIR
    
    # If we're appending a dir, it's coming from the post processor which is
    # already bytestring
    if not append:
        dir = dir.encode(headphones.SYS_ENCODING)
        
    if not os.path.isdir(dir):
        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING))
        return

    myDB = db.DBConnection()
    
    if not append:
        # Clean up bad filepaths
        tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL')
    
        for track in tracks:
            if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)):
                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']])

        myDB.action('DELETE from have')

    logger.info('Scanning music directory: %s' % dir)

    new_artists = []
    bitrates = []
    
    song_list = []
    
    for r,d,f in os.walk(dir):
        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):

                song = os.path.join(r, files)

                # We need the unicode path to use for logging, inserting into database
                unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace')

                # Try to read the metadata
                try:
                    f = MediaFile(song)

                except:
                    logger.error('Cannot read file: ' + unicode_song_path)
                    continue
                    
                # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)
                    
                # Use the album artist over the artist if available
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    f_artist = None
                    
                # Add the song to our song list - 
                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)

                song_dict = { 'TrackID' : f.mb_trackid,
                              'ReleaseID' : f.mb_albumid,
                              'ArtistName' : f_artist,
                              'AlbumTitle' : f.album,
                              'TrackNumber': f.track,
                              'TrackLength': f.length,
                              'Genre'      : f.genre,
                              'Date'       : f.date,
                              'TrackTitle' : f.title,
                              'BitRate'    : f.bitrate,
                              'Format'     : f.format,
                              'Location'   : unicode_song_path }
                              
                song_list.append(song_dict)

    # Now we start track matching
    total_number_of_songs = len(song_list)
    logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir + "'. Matching tracks to the appropriate releases....")
    
    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
    
    song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
    
    # We'll use this to give a % completion, just because the track matching might take a while
    song_count = 0
    
    for song in song_list:
        
        song_count += 1
        completion_percentage = float(song_count)/total_number_of_songs * 100
        
        if completion_percentage%10 == 0:
            logger.info("Track matching is " + str(completion_percentage) + "% complete")
        
        # If the track has a trackid & releaseid (beets: albumid) that the most surefire way
        # of identifying a track to a specific release so we'll use that first
        if song['TrackID'] and song['ReleaseID']:

            # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet
            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
            
            # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table
            if not track:
                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
    
            if track:
                # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo
                controlValueDict = { 'TrackID'   : track['TrackID'],
                                     'ReleaseID' : track['ReleaseID'] }
                
                # Insert it into the Headphones hybrid release (ReleaseID == AlbumID)                   
                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
                                           'ReleaseID' : track['AlbumID'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }
                                 
                # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict
                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)
                
                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
                
                # Matched. Move on to the next one:
                continue
    
        # If we can't find it with TrackID & ReleaseID, next most specific will be 
        # releaseid + tracktitle, although perhaps less reliable due to a higher 
        # likelihood of variations in the song title (e.g. feat. artists)
        if song['ReleaseID'] and song['TrackTitle']:
    
            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
    
            if not track:
                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
                
            if track:
                # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table)
                # So use both TrackID & ReleaseID as the control values
                controlValueDict = { 'TrackID'   : track['TrackID'],
                                     'ReleaseID' : track['ReleaseID'] }
                                     
                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
                                           'ReleaseID' : track['AlbumID'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }
                                 
                # Update both tables here as well
                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)
                
                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
                
                # Done
                continue
                
        # Next most specific will be the opposite: a TrackID and an AlbumTitle
        # TrackIDs span multiple releases so if something is on an official album
        # and a compilation, for example, this will match it to the right one
        # However - there may be multiple matches here
        if song['TrackID'] and song['AlbumTitle']:
    
            # Even though there might be multiple matches, we just need to grab one to confirm a match
            track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
    
            if not track:
                track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
                
            if track:
                # Don't need the hybridControlValueDict here since ReleaseID is not unique
                controlValueDict = { 'TrackID'   : track['TrackID'],
                                     'AlbumTitle' : track['AlbumTitle'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }

                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)

                continue   
        
        # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably 
        # even more unreliable than the previous queries, and might span multiple releases)
        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
            
            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
    
            if not track:
                track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
                
            if track:
                controlValueDict = { 'ArtistName' : track['ArtistName'],
                                     'AlbumTitle' : track['AlbumTitle'],
                                     'TrackTitle' : track['TrackTitle'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }

                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)

                continue
        
        # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc)
        # This is more reliable than the former but requires some string manipulation so we'll do it only
        # if we can't find a match with the original data
        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
            
            CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
            
            track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
            
            if not track:
                track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
    
            if track:
                controlValueDict = { 'CleanName' : track['CleanName'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }

                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)

                continue     
        
        # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable
        # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track
        # with more specific information exists in the library, it'll overwrite these values
        if song['TrackID']:
    
            track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone()
            
            if not track:
                track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone()
    
            if track:
                controlValueDict = { 'TrackID' : track['TrackID'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }

                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)

                continue          
        
        # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
        if song['ArtistName']:
            new_artists.append(song['ArtistName'])
        else:
            continue
        
        # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database                
        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
            CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
        else:
            continue
        
        myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])

    logger.info('Completed matching tracks from directory: %s' % dir)
    
    
    if not append:
        # Clean up the new artist list
        unique_artists = {}.fromkeys(new_artists).keys()
        current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
        
        artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]]
        
        # Update track counts
        logger.info('Updating current artist track counts')
    
        for artist in current_artists:
            # Have tracks are selected from tracks table and not all tracks because of duplicates
            # We update the track count upon an album switch to compliment this
            havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']]))
            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']])
            
        logger.info('Found %i new artists' % len(artist_list))
    
        if len(artist_list):
            if headphones.ADD_ARTISTS:
                logger.info('Importing %i new artists' % len(artist_list))
                importer.artistlist_to_mbids(artist_list)
            else:
                logger.info('To add these artists, go to Manage->Manage New Artists')
                myDB.action('DELETE from newartists')
                for artist in artist_list:
                    myDB.action('INSERT into newartists VALUES (?)', [artist])
        
        if headphones.DETECT_BITRATE:
            headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
            
    else:
        # If we're appending a new album to the database, update the artists total track counts
        logger.info('Updating artist track counts')
        
        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName]))
        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
Exemple #2
0
def addArtisttoDB(artistid, extrasonly=False):
    
    # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
    from headphones import cache
    
    # Can't add various artists - throws an error from MB
    if artistid == various_artists_mbid:
        logger.warn('Cannot import Various Artists.')
        return
        
    myDB = db.DBConnection()
    
    # Delete from blacklist if it's on there
    myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])

    # We need the current minimal info in the database instantly
    # so we don't throw a 500 error when we redirect to the artistPage

    controlValueDict = {"ArtistID":     artistid}

    # Don't replace a known artist name with an "Artist ID" placeholder

    dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone()
    if dbartist is None:
        newValueDict = {"ArtistName":   "Artist ID: %s" % (artistid),
                "Status":   "Loading"}
    else:
        newValueDict = {"Status":   "Loading"}

    myDB.upsert("artists", newValueDict, controlValueDict)
        
    artist = mb.getArtist(artistid, extrasonly)
    
    if not artist:
        logger.warn("Error fetching artist info. ID: " + artistid)
        if dbartist is None:
            newValueDict = {"ArtistName":   "Fetch failed, try refreshing. (%s)" % (artistid),
                    "Status":   "Active"}
        else:
            newValueDict = {"Status":   "Active"}
        myDB.upsert("artists", newValueDict, controlValueDict)
        return
    
    if artist['artist_name'].startswith('The '):
        sortname = artist['artist_name'][4:]
    else:
        sortname = artist['artist_name']
        

    logger.info(u"Now adding/updating: " + artist['artist_name'])
    controlValueDict = {"ArtistID":     artistid}
    newValueDict = {"ArtistName":       artist['artist_name'],
                    "ArtistSortName":   sortname,
                    "DateAdded":        helpers.today(),
                    "Status":           "Loading"}
    
    if headphones.INCLUDE_EXTRAS:
        newValueDict['IncludeExtras'] = 1
    
    myDB.upsert("artists", newValueDict, controlValueDict)

    for rg in artist['releasegroups']:
        
        rgid = rg['id']
        
        # check if the album already exists
        rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rg['id']])
                    
        try:    
            release_dict = mb.getReleaseGroup(rgid)
        except Exception, e:
            logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title'])
            continue
            
        if not release_dict:
            continue
    
        logger.info(u"Now adding/updating album: " + rg['title'])

        controlValueDict = {"AlbumID":  rg['id']}

        newValueDict = {"ArtistID":         artistid,
                        "ArtistName":       artist['artist_name'],
                        "AlbumTitle":       rg['title'],
                        "AlbumASIN":        release_dict['asin'],
                        "ReleaseDate":      release_dict['releasedate'],
                        "Type":             rg['type']
                        }
        
        # Only change the status & add DateAdded if the album is not already in the database
        if not len(rg_exists):

            newValueDict['DateAdded']= helpers.today()
                            
            if headphones.AUTOWANT_ALL:
                newValueDict['Status'] = "Wanted"
            elif release_dict['releasedate'] > helpers.today() and headphones.AUTOWANT_UPCOMING:
                newValueDict['Status'] = "Wanted"
            else:
                newValueDict['Status'] = "Skipped"
        
        myDB.upsert("albums", newValueDict, controlValueDict)

        for track in release_dict['tracks']:
        
            cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
        
            controlValueDict = {"TrackID":  track['id'],
                                "AlbumID":  rg['id']}

            newValueDict = {"ArtistID":     artistid,
                        "ArtistName":       artist['artist_name'],
                        "AlbumTitle":       rg['title'],
                        "AlbumASIN":        release_dict['asin'],
                        "TrackTitle":       track['title'],
                        "TrackDuration":    track['duration'],
                        "TrackNumber":      track['number'],
                        "CleanName":        cleanname
                        }
            
            match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
            
            if not match:
                match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
            if not match:
                match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
            if match:
                newValueDict['Location'] = match['Location']
                newValueDict['BitRate'] = match['BitRate']
                newValueDict['Format'] = match['Format']
                myDB.action('DELETE from have WHERE Location=?', [match['Location']])
                
            myDB.upsert("tracks", newValueDict, controlValueDict)
            
        logger.debug(u"Updating album cache for " + rg['title'])
        cache.getThumb(AlbumID=rg['id'])
Exemple #3
0
def addArtisttoDB(artistid, extrasonly=False, forcefull=False):

    # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
    from headphones import cache

    # Can't add various artists - throws an error from MB
    if artistid in blacklisted_special_artists:
        logger.warn("Cannot import blocked special purpose artist with id" + artistid)
        return

    # We'll use this to see if we should update the 'LastUpdated' time stamp
    errors = False

    myDB = db.DBConnection()

    # Delete from blacklist if it's on there
    myDB.action("DELETE from blacklist WHERE ArtistID=?", [artistid])

    # We need the current minimal info in the database instantly
    # so we don't throw a 500 error when we redirect to the artistPage

    controlValueDict = {"ArtistID": artistid}

    # Don't replace a known artist name with an "Artist ID" placeholder

    dbartist = myDB.action("SELECT * FROM artists WHERE ArtistID=?", [artistid]).fetchone()

    # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
    if not dbartist:
        newValueDict = {
            "ArtistName": "Artist ID: %s" % (artistid),
            "Status": "Loading",
            "IncludeExtras": headphones.INCLUDE_EXTRAS,
            "Extras": headphones.EXTRAS,
        }
    else:
        newValueDict = {"Status": "Loading"}

    myDB.upsert("artists", newValueDict, controlValueDict)

    artist = mb.getArtist(artistid, extrasonly)

    if artist and artist.get("artist_name") in blacklisted_special_artist_names:
        logger.warn("Cannot import blocked special purpose artist: %s" % artist.get("artist_name"))
        myDB.action("DELETE from artists WHERE ArtistID=?", [artistid])
        # in case it's already in the db
        myDB.action("DELETE from albums WHERE ArtistID=?", [artistid])
        myDB.action("DELETE from tracks WHERE ArtistID=?", [artistid])
        return

    if not artist:
        logger.warn("Error fetching artist info. ID: " + artistid)
        if dbartist is None:
            newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), "Status": "Active"}
        else:
            newValueDict = {"Status": "Active"}
        myDB.upsert("artists", newValueDict, controlValueDict)
        return

    if artist["artist_name"].startswith("The "):
        sortname = artist["artist_name"][4:]
    else:
        sortname = artist["artist_name"]

    logger.info(u"Now adding/updating: " + artist["artist_name"])
    controlValueDict = {"ArtistID": artistid}
    newValueDict = {
        "ArtistName": artist["artist_name"],
        "ArtistSortName": sortname,
        "DateAdded": helpers.today(),
        "Status": "Loading",
    }

    myDB.upsert("artists", newValueDict, controlValueDict)

    # See if we need to grab extras. Artist specific extras take precedence over global option
    # Global options are set when adding a new artist
    myDB = db.DBConnection()

    try:
        db_artist = myDB.action("SELECT IncludeExtras, Extras from artists WHERE ArtistID=?", [artistid]).fetchone()
        includeExtras = db_artist["IncludeExtras"]
    except IndexError:
        includeExtras = False

    # Clean all references to release group in dB that are no longer referenced in musicbrainz
    group_list = []
    force_repackage = 0
    # Don't nuke the database if there's a MusicBrainz error
    if len(artist["releasegroups"]) != 0 and not extrasonly:
        for groups in artist["releasegroups"]:
            group_list.append(groups["id"])
        remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
        for items in remove_missing_groups_from_albums:
            if items["AlbumID"] not in group_list:
                # Remove all from albums/tracks that aren't in release groups
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [items["AlbumID"]])
                myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items["AlbumID"]])
                myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items["AlbumID"]])
                myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items["AlbumID"]])
                logger.info(
                    "[%s] Removing all references to release group %s to reflect MusicBrainz"
                    % (artist["artist_name"], items["AlbumID"])
                )
                force_repackage = 1
    else:
        logger.info("[%s] Error pulling data from MusicBrainz:  Maintaining dB" % artist["artist_name"])

    # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks

    for rg in artist["releasegroups"]:

        al_title = rg["title"]
        today = helpers.today()
        rgid = rg["id"]
        skip_log = 0
        # Make a user configurable variable to skip update of albums with release dates older than this date (in days)
        pause_delta = headphones.MB_IGNORE_AGE

        check_release_date = myDB.action(
            "SELECT ReleaseDate, Status from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)
        ).fetchone()

        # Skip update if Status set
        if check_release_date and check_release_date[1]:
            logger.info(
                "[%s] Not updating: %s (Status is %s, skipping)"
                % (artist["artist_name"], rg["title"], check_release_date[1])
            )
            continue

        if not forcefull:
            if check_release_date:
                if check_release_date[0] is None:
                    logger.info("[%s] Now updating: %s (No Release Date)" % (artist["artist_name"], rg["title"]))
                    new_releases = mb.get_new_releases(rgid, includeExtras, True)
                else:
                    if len(check_release_date[0]) == 10:
                        release_date = check_release_date[0]
                    elif len(check_release_date[0]) == 7:
                        release_date = check_release_date[0] + "-31"
                    elif len(check_release_date[0]) == 4:
                        release_date = check_release_date[0] + "-12-31"
                    else:
                        release_date = today
                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
                        logger.info(
                            "[%s] Now updating: %s (Release Date <%s Days) "
                            % (artist["artist_name"], rg["title"], pause_delta)
                        )
                        new_releases = mb.get_new_releases(rgid, includeExtras, True)
                    else:
                        logger.info(
                            "[%s] Skipping: %s (Release Date >%s Days)"
                            % (artist["artist_name"], rg["title"], pause_delta)
                        )
                        skip_log = 1
                        new_releases = 0
            else:
                logger.info("[%s] Now adding: %s (New Release Group)" % (artist["artist_name"], rg["title"]))
                new_releases = mb.get_new_releases(rgid, includeExtras)

            if force_repackage == 1:
                new_releases = -1
                logger.info("[%s] Forcing repackage of %s (Release Group Removed)" % (artist["artist_name"], al_title))
            else:
                new_releases = new_releases
        else:
            logger.info("[%s] Now adding/updating: %s (Comprehensive Force)" % (artist["artist_name"], rg["title"]))
            new_releases = mb.get_new_releases(rgid, includeExtras, forcefull)

        # What this does is adds new releases per artist to the allalbums + alltracks databases
        # new_releases = mb.get_new_releases(rgid,includeExtras)
        # print al_title
        # print new_releases

        if new_releases != 0:
            # Dump existing hybrid release since we're repackaging/replacing it
            myDB.action("DELETE from albums WHERE ReleaseID=?", [rg["id"]])
            myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg["id"]])
            myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg["id"]])
            myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg["id"]])
            # This will be used later to build a hybrid release
            fullreleaselist = []
            # Search for releases within a release group
            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg["id"]])
            # Build the dictionary for the fullreleaselist
            for items in find_hybrid_releases:
                if (
                    items["ReleaseID"] != rg["id"]
                ):  # don't include hybrid information, since that's what we're replacing
                    hybrid_release_id = items["ReleaseID"]
                    newValueDict = {
                        "ArtistID": items["ArtistID"],
                        "ArtistName": items["ArtistName"],
                        "AlbumTitle": items["AlbumTitle"],
                        "AlbumID": items["AlbumID"],
                        "AlbumASIN": items["AlbumASIN"],
                        "ReleaseDate": items["ReleaseDate"],
                        "Type": items["Type"],
                        "ReleaseCountry": items["ReleaseCountry"],
                        "ReleaseFormat": items["ReleaseFormat"],
                    }
                    find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id])
                    totalTracks = 1
                    hybrid_track_array = []
                    for hybrid_tracks in find_hybrid_tracks:
                        hybrid_track_array.append(
                            {
                                "number": hybrid_tracks["TrackNumber"],
                                "title": hybrid_tracks["TrackTitle"],
                                "id": hybrid_tracks["TrackID"],
                                #'url':           hybrid_tracks['TrackURL'],
                                "duration": hybrid_tracks["TrackDuration"],
                            }
                        )
                        totalTracks += 1
                    newValueDict["ReleaseID"] = hybrid_release_id
                    newValueDict["Tracks"] = hybrid_track_array
                    fullreleaselist.append(newValueDict)

                # print fullreleaselist

            # Basically just do the same thing again for the hybrid release
            # This may end up being called with an empty fullreleaselist
            try:
                hybridrelease = getHybridRelease(fullreleaselist)
                logger.info("[%s] Packaging %s releases into hybrid title" % (artist["artist_name"], rg["title"]))
            except Exception, e:
                errors = True
                logger.warn(
                    "[%s] Unable to get hybrid release information for %s: %s" % (artist["artist_name"], rg["title"], e)
                )
                continue

            # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
            # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
            # The hybrid won't have a country or a format
            controlValueDict = {"ReleaseID": rg["id"]}

            newValueDict = {
                "ArtistID": artistid,
                "ArtistName": artist["artist_name"],
                "AlbumTitle": rg["title"],
                "AlbumID": rg["id"],
                "AlbumASIN": hybridrelease["AlbumASIN"],
                "ReleaseDate": hybridrelease["ReleaseDate"],
                "Type": rg["type"],
            }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            for track in hybridrelease["Tracks"]:

                cleanname = helpers.cleanName(artist["artist_name"] + " " + rg["title"] + " " + track["title"])

                controlValueDict = {"TrackID": track["id"], "ReleaseID": rg["id"]}

                newValueDict = {
                    "ArtistID": artistid,
                    "ArtistName": artist["artist_name"],
                    "AlbumTitle": rg["title"],
                    "AlbumASIN": hybridrelease["AlbumASIN"],
                    "AlbumID": rg["id"],
                    "TrackTitle": track["title"],
                    "TrackDuration": track["duration"],
                    "TrackNumber": track["number"],
                    "CleanName": cleanname,
                }

                match = myDB.action(
                    "SELECT Location, BitRate, Format from have WHERE CleanName=?", [cleanname]
                ).fetchone()

                if not match:
                    match = myDB.action(
                        "SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?",
                        [artist["artist_name"], rg["title"], track["title"]],
                    ).fetchone()
                # if not match:
                # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                if match:
                    newValueDict["Location"] = match["Location"]
                    newValueDict["BitRate"] = match["BitRate"]
                    newValueDict["Format"] = match["Format"]
                    # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                    myDB.action("UPDATE have SET Matched=? WHERE Location=?", (rg["id"], match["Location"]))

                myDB.upsert("alltracks", newValueDict, controlValueDict)

            # Delete matched tracks from the have table
            # myDB.action('DELETE from have WHERE Matched="True"')

            # If there's no release in the main albums tables, add the default (hybrid)
            # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
            # check if the album already exists
            rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg["id"]]).fetchone()
            if not rg_exists:
                releaseid = rg["id"]
            else:
                releaseid = rg_exists["ReleaseID"]

            album = myDB.action("SELECT * from allalbums WHERE ReleaseID=?", [releaseid]).fetchone()

            controlValueDict = {"AlbumID": rg["id"]}

            newValueDict = {
                "ArtistID": album["ArtistID"],
                "ArtistName": album["ArtistName"],
                "AlbumTitle": album["AlbumTitle"],
                "ReleaseID": album["ReleaseID"],
                "AlbumASIN": album["AlbumASIN"],
                "ReleaseDate": album["ReleaseDate"],
                "Type": album["Type"],
                "ReleaseCountry": album["ReleaseCountry"],
                "ReleaseFormat": album["ReleaseFormat"],
            }

            if not rg_exists:

                today = helpers.today()

                newValueDict["DateAdded"] = today

                if headphones.AUTOWANT_ALL:
                    newValueDict["Status"] = "Wanted"
                elif album["ReleaseDate"] > today and headphones.AUTOWANT_UPCOMING:
                    newValueDict["Status"] = "Wanted"
                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
                # The first test just makes sure we have year-month-day
                elif (
                    helpers.get_age(album["ReleaseDate"])
                    and helpers.get_age(today) - helpers.get_age(album["ReleaseDate"]) < 21
                    and headphones.AUTOWANT_UPCOMING
                ):
                    newValueDict["Status"] = "Wanted"
                else:
                    newValueDict["Status"] = "Skipped"

            myDB.upsert("albums", newValueDict, controlValueDict)

            tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [releaseid]).fetchall()

            # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT
            total_track_count = len(tracks)

            for track in tracks:

                controlValueDict = {"TrackID": track["TrackID"], "AlbumID": rg["id"]}

                newValueDict = {
                    "ArtistID": track["ArtistID"],
                    "ArtistName": track["ArtistName"],
                    "AlbumTitle": track["AlbumTitle"],
                    "AlbumASIN": track["AlbumASIN"],
                    "ReleaseID": track["ReleaseID"],
                    "TrackTitle": track["TrackTitle"],
                    "TrackDuration": track["TrackDuration"],
                    "TrackNumber": track["TrackNumber"],
                    "CleanName": track["CleanName"],
                    "Location": track["Location"],
                    "Format": track["Format"],
                    "BitRate": track["BitRate"],
                }

                myDB.upsert("tracks", newValueDict, controlValueDict)

            # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
            have_track_count = len(
                myDB.select("SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL", [rg["id"]])
            )
            marked_as_downloaded = False

            if rg_exists:
                if rg_exists["Status"] == "Skipped" and (
                    (have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0)
                ):
                    myDB.action("UPDATE albums SET Status=? WHERE AlbumID=?", ["Downloaded", rg["id"]])
                    marked_as_downloaded = True
            else:
                if (have_track_count / float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT / 100.0):
                    myDB.action("UPDATE albums SET Status=? WHERE AlbumID=?", ["Downloaded", rg["id"]])
                    marked_as_downloaded = True

            logger.info(u"[%s] Seeing if we need album art for %s" % (artist["artist_name"], rg["title"]))
            cache.getThumb(AlbumID=rg["id"])

            # start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected:
            if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL:
                from headphones import searcher

                searcher.searchforalbum(albumid=rg["id"])
        else:
            if skip_log == 0:
                logger.info(u"[%s] No new releases, so no changes made to %s" % (artist["artist_name"], rg["title"]))
Exemple #4
0
                "ArtistName": release['ArtistName'],
                "AlbumTitle": release['AlbumTitle'],
                "AlbumID": release['AlbumID'],
                "AlbumASIN": release['AlbumASIN'],
                "ReleaseDate": release['ReleaseDate'],
                "Type": release['Type'],
                "ReleaseCountry": release['ReleaseCountry'],
                "ReleaseFormat": release['ReleaseFormat']
            }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            for track in release['Tracks']:

                cleanname = helpers.cleanName(release['ArtistName'] + ' ' +
                                              release['AlbumTitle'] + ' ' +
                                              track['title'])

                controlValueDict = {
                    "TrackID": track['id'],
                    "ReleaseID": release['ReleaseID']
                }

                newValueDict = {
                    "ArtistID": release['ArtistID'],
                    "ArtistName": release['ArtistName'],
                    "AlbumTitle": release['AlbumTitle'],
                    "AlbumID": release['AlbumID'],
                    "AlbumASIN": release['AlbumASIN'],
                    "TrackTitle": track['title'],
                    "TrackDuration": track['duration'],
Exemple #5
0
def addReleaseById(rid, rgid=None):

    myDB = db.DBConnection()

    # Create minimum info upfront if added from searchresults
    status = ''
    if rgid:
        dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid])
        if not dbalbum:
            status = 'Loading'
            controlValueDict = {"AlbumID": rgid}
            newValueDict = {"AlbumTitle": rgid,
                            "ArtistName": status,
                            "Status": status}
            myDB.upsert("albums", newValueDict, controlValueDict)
            time.sleep(1)

    rgid = None
    artistid = None
    release_dict = None
    results = myDB.select("SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid])
    for result in results:
        rgid = result['ReleaseGroupID']
        artistid = result['ArtistID']
        logger.debug("Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid)
    if not rgid:
        #didn't find it in the cache, get the information from MB
        logger.debug("Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID")
        try:
            release_dict = mb.getRelease(rid)
        except Exception as e:
            logger.info('Unable to get release information for Release %s: %s', rid, e)
            if status == 'Loading':
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
            return
        if not release_dict:
            logger.info('Unable to get release information for Release %s: no dict', rid)
            if status == 'Loading':
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
            return

        rgid = release_dict['rgid']
        artistid = release_dict['artist_id']

    #we don't want to make more calls to MB here unless we have to, could be happening quite a lot
    rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid])

    #make sure the artist exists since I don't know what happens later if it doesn't
    artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid])

    if not artist_exists and release_dict:
        if release_dict['artist_name'].startswith('The '):
            sortname = release_dict['artist_name'][4:]
        else:
            sortname = release_dict['artist_name']

        logger.info(u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
        controlValueDict = {"ArtistID": release_dict['artist_id']}
        newValueDict = {"ArtistName": release_dict['artist_name'],
                        "ArtistSortName": sortname,
                        "DateAdded": helpers.today(),
                        "Status": "Paused"}

        if headphones.CONFIG.INCLUDE_EXTRAS:
            newValueDict['IncludeExtras'] = 1
            newValueDict['Extras'] = headphones.CONFIG.EXTRAS

        myDB.upsert("artists", newValueDict, controlValueDict)

    elif not artist_exists and not release_dict:
        logger.error("Artist does not exist in the database and did not get a valid response from MB. Skipping release.")
        if status == 'Loading':
            myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
        return

    if not rg_exists and release_dict or status == 'Loading' and release_dict:  #it should never be the case that we have an rg and not the artist
                                                                                #but if it is this will fail
        logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
        controlValueDict = {"AlbumID": rgid}
        if status != 'Loading':
            status = 'Wanted'

        newValueDict = {"ArtistID": release_dict['artist_id'],
                        "ReleaseID": rgid,
                        "ArtistName": release_dict['artist_name'],
                        "AlbumTitle": release_dict['title'] if 'title' in release_dict else release_dict['rg_title'],
                        "AlbumASIN": release_dict['asin'],
                        "ReleaseDate": release_dict['date'],
                        "DateAdded": helpers.today(),
                        "Status": status,
                        "Type": release_dict['rg_type'],
                        "ReleaseID": rid
                        }

        myDB.upsert("albums", newValueDict, controlValueDict)

        #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB
        myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']])

        for track in release_dict['tracks']:
            cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title'])

            controlValueDict = {"TrackID": track['id'],
                                "AlbumID": rgid}
            newValueDict = {"ArtistID": release_dict['artist_id'],
                        "ArtistName": release_dict['artist_name'],
                        "AlbumTitle": release_dict['rg_title'],
                        "AlbumASIN": release_dict['asin'],
                        "TrackTitle": track['title'],
                        "TrackDuration": track['duration'],
                        "TrackNumber": track['number'],
                        "CleanName": cleanname
                        }

            match = myDB.action('SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?', [cleanname]).fetchone()

            if not match:
                match = myDB.action('SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone()

            #if not match:
                #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()

            if match:
                newValueDict['Location'] = match['Location']
                newValueDict['BitRate'] = match['BitRate']
                newValueDict['Format'] = match['Format']
                #myDB.action('DELETE from have WHERE Location=?', [match['Location']])

                # If the album has been scanned before adding the release it will be unmatched, update to matched
                if match['Matched'] == 'Failed':
                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release_dict['rgid'], match['Location']))

            myDB.upsert("tracks", newValueDict, controlValueDict)

        # Reset status
        if status == 'Loading':
            controlValueDict = {"AlbumID": rgid}
            if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED:
                newValueDict = {"Status": "Wanted"}
            else:
                newValueDict = {"Status": "Skipped"}
            myDB.upsert("albums", newValueDict, controlValueDict)

        # Start a search for the album
        if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED:
            import searcher
            searcher.searchforalbum(rgid, False)

    elif not rg_exists and not release_dict:
        logger.error("ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.")
        if status == 'Loading':
            myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
        return
    else:
        logger.info('Release ' + str(rid) + " already exists in the database!")
Exemple #6
0
def addArtisttoDB(artistid, extrasonly=False):
    
    # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
    from headphones import cache
    
    # Can't add various artists - throws an error from MB
    if artistid == various_artists_mbid:
        logger.warn('Cannot import Various Artists.')
        return
        
    # We'll use this to see if we should update the 'LastUpdated' time stamp
    errors = False
        
    myDB = db.DBConnection()
    
    # Delete from blacklist if it's on there
    myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])

    # We need the current minimal info in the database instantly
    # so we don't throw a 500 error when we redirect to the artistPage

    controlValueDict = {"ArtistID":     artistid}

    # Don't replace a known artist name with an "Artist ID" placeholder

    dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone()
    
    # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
    if not dbartist:
        newValueDict = {"ArtistName":   "Artist ID: %s" % (artistid),
                        "Status":       "Loading",
                        "IncludeExtras": headphones.INCLUDE_EXTRAS,
                        "Extras":        headphones.EXTRAS }
    else:
        newValueDict = {"Status":   "Loading"}

    myDB.upsert("artists", newValueDict, controlValueDict)
        
    artist = mb.getArtist(artistid, extrasonly)
    
    if not artist:
        logger.warn("Error fetching artist info. ID: " + artistid)
        if dbartist is None:
            newValueDict = {"ArtistName":   "Fetch failed, try refreshing. (%s)" % (artistid),
                    "Status":   "Active"}
        else:
            newValueDict = {"Status":   "Active"}
        myDB.upsert("artists", newValueDict, controlValueDict)
        return
    
    if artist['artist_name'].startswith('The '):
        sortname = artist['artist_name'][4:]
    else:
        sortname = artist['artist_name']
        

    logger.info(u"Now adding/updating: " + artist['artist_name'])
    controlValueDict = {"ArtistID":     artistid}
    newValueDict = {"ArtistName":       artist['artist_name'],
                    "ArtistSortName":   sortname,
                    "DateAdded":        helpers.today(),
                    "Status":           "Loading"}
    
    myDB.upsert("artists", newValueDict, controlValueDict)

    for rg in artist['releasegroups']:
        
        logger.info("Now adding/updating: " + rg['title'])
        
        rgid = rg['id']
        
        # check if the album already exists
        rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
                    
        try:    
            releaselist = mb.getReleaseGroup(rgid)
        except Exception, e:
            logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title'])
            continue
            
        if not releaselist:
            errors = True
            continue
        
        # This will be used later to build a hybrid release     
        fullreleaselist = []
            
        for release in releaselist:
        # What we're doing here now is first updating the allalbums & alltracks table to the most
        # current info, then moving the appropriate release into the album table and its associated
        # tracks into the tracks table
            
            releaseid = release['id']
            
            try:
                releasedict = mb.getRelease(releaseid, include_artist_info=False)
            except Exception, e:
                errors = True
                logger.info('Unable to get release information for %s: %s' % (release['id'], e))
                continue
           
            if not releasedict:
                errors = True
                continue

            controlValueDict = {"ReleaseID":  release['id']}

            newValueDict = {"ArtistID":         artistid,
                            "ArtistName":       artist['artist_name'],
                            "AlbumTitle":       rg['title'],
                            "AlbumID":          rg['id'],
                            "AlbumASIN":        releasedict['asin'],
                            "ReleaseDate":      releasedict['date'],
                            "Type":             rg['type'],
                            "ReleaseCountry":   releasedict['country'],
                            "ReleaseFormat":    releasedict['format']
                        }
                        
            myDB.upsert("allalbums", newValueDict, controlValueDict)
            
            # Build the dictionary for the fullreleaselist
            newValueDict['ReleaseID'] = release['id']
            newValueDict['Tracks'] = releasedict['tracks']
            fullreleaselist.append(newValueDict)
            
            for track in releasedict['tracks']:

                cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
        
                controlValueDict = {"TrackID":      track['id'],
                                    "ReleaseID":    release['id']}

                newValueDict = {"ArtistID":         artistid,
                                "ArtistName":       artist['artist_name'],
                                "AlbumTitle":       rg['title'],
                                "AlbumASIN":        releasedict['asin'],
                                "AlbumID":          rg['id'],
                                "TrackTitle":       track['title'],
                                "TrackDuration":    track['duration'],
                                "TrackNumber":      track['number'],
                                "CleanName":        cleanname
                            }
                            
                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
            
                if not match:
                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
                if not match:
                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
                if match:
                    newValueDict['Location'] = match['Location']
                    newValueDict['BitRate'] = match['BitRate']
                    newValueDict['Format'] = match['Format']
                    myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                                
                myDB.upsert("alltracks", newValueDict, controlValueDict)
def libraryScan(dir=None):

	if not dir:
		dir = headphones.MUSIC_DIR
		
	try:
		dir = str(dir)
	except UnicodeEncodeError:
		dir = unicode(dir).encode('unicode_escape')
		
	if not os.path.isdir(dir):
		logger.warn('Cannot find directory: %s. Not scanning' % dir)
		return

	myDB = db.DBConnection()
	
	# Clean up bad filepaths
	tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL')
	
	for track in tracks:
		if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)):
			myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']])

	logger.info('Scanning music directory: %s' % dir)

	new_artists = []
	bitrates = []

	myDB.action('DELETE from have')
	
	for r,d,f in os.walk(dir):
		for files in f:
			# MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
			if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):

				song = os.path.join(r, files)
				file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors='replace')

				# Try to read the metadata
				try:
					f = MediaFile(song)

				except:
					logger.error('Cannot read file: ' + file)
					continue
					
				# Grab the bitrates for the auto detect bit rate option
				if f.bitrate:
					bitrates.append(f.bitrate)
				
				# Try to find a match based on artist/album/tracktitle
				if f.albumartist:
					f_artist = f.albumartist
				elif f.artist:
					f_artist = f.artist
				else:
					continue
				
				if f_artist and f.album and f.title:

					track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone()
						
					if not track:
						track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone()
					
					if track:
						myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']])
						continue		
				
				# Try to match on mbid if available and we couldn't find a match based on metadata
				if f.mb_trackid:

					# Wondering if theres a better way to do this -> do one thing if the row exists,
					# do something else if it doesn't
					track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone()
		
					if track:
						myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [file, f.bitrate, f.format, track['TrackID']])
						continue				
				
				# if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
				new_artists.append(f_artist)
				
				# The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database				
				myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist+' '+f.album+' '+f.title), f.format])

	logger.info('Completed scanning of directory: %s' % dir)
	logger.info('Checking filepaths to see if we can find any matches')

	# Now check empty file paths to see if we can find a match based on their folder format
	tracks = myDB.select('SELECT * from tracks WHERE Location IS NULL')
	for track in tracks:
	
		release = myDB.action('SELECT * from albums WHERE AlbumID=?', [track['AlbumID']]).fetchone()

		try:
			year = release['ReleaseDate'][:4]
		except TypeError:
			year = ''
			
		artist = release['ArtistName'].replace('/', '_')
		album = release['AlbumTitle'].replace('/', '_')
	
		if release['ArtistName'].startswith('The '):
			sortname = release['ArtistName'][4:]
		else:
			sortname = release['ArtistName']
		
		if sortname.isdigit():
			firstchar = '0-9'
		else:
			firstchar = sortname[0]
			
		lowerfirst = firstchar.lower()
		
		albumvalues = {	'artist':	artist,
						'album':	album,
						'year':		year,
						'first':	firstchar,
						'lowerfirst':	lowerfirst
					}
				
		
		folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues)
		folder = folder.replace('./', '_/').replace(':','_').replace('?','_')
		
		if folder.endswith('.'):
			folder = folder.replace(folder[len(folder)-1], '_')

		if not track['TrackNumber']:
			tracknumber = ''
		else:
			tracknumber = '%02d' % track['TrackNumber']
			
		trackvalues = {	'tracknumber':	tracknumber,
						'title':		track['TrackTitle'],
						'artist':		release['ArtistName'],
						'album':		release['AlbumTitle'],
						'year':			year
						}
		
		new_file_name = helpers.replace_all(headphones.FILE_FORMAT, trackvalues).replace('/','_') + '.*'
		
		new_file_name = new_file_name.replace('?','_').replace(':', '_')
		
		full_path_to_file = os.path.normpath(os.path.join(headphones.MUSIC_DIR, folder, new_file_name)).encode(headphones.SYS_ENCODING, 'replace')

		match = glob.glob(full_path_to_file)
		
		if match:

			logger.info('Found a match: %s. Writing MBID to metadata' % match[0])
			
			unipath = unicode(match[0], headphones.SYS_ENCODING, errors='replace')

			myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [unipath, track['TrackID']])
			myDB.action('DELETE from have WHERE Location=?', [unipath])
			
			# Try to insert the appropriate track id so we don't have to keep doing this
			try:
				f = MediaFile(match[0])
				f.mb_trackid = track['TrackID']
				f.save()
				myDB.action('UPDATE tracks SET BitRate=?, Format=? WHERE TrackID=?', [f.bitrate, f.format, track['TrackID']])

				logger.debug('Wrote mbid to track: %s' % match[0])

			except:
				logger.error('Error embedding track id into: %s' % match[0])
				continue

	logger.info('Done checking empty filepaths')
	logger.info('Done syncing library with directory: %s' % dir)
	
	# Clean up the new artist list
	unique_artists = {}.fromkeys(new_artists).keys()
	current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
	
	artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]]
	
	# Update track counts
	logger.info('Updating track counts')

	for artist in current_artists:
		havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']]))
		myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']])
		
	logger.info('Found %i new artists' % len(artist_list))

	if len(artist_list):
		if headphones.ADD_ARTISTS:
			logger.info('Importing %i new artists' % len(artist_list))
			importer.artistlist_to_mbids(artist_list)
		else:
			logger.info('To add these artists, go to Manage->Manage New Artists')
			headphones.NEW_ARTISTS = artist_list
	
	if headphones.DETECT_BITRATE:
		headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
Exemple #8
0
            newValueDict = {"ArtistID":         release['ArtistID'],
                            "ArtistName":       release['ArtistName'],
                            "AlbumTitle":       release['AlbumTitle'],
                            "AlbumID":          release['AlbumID'],
                            "AlbumASIN":        release['AlbumASIN'],
                            "ReleaseDate":      release['ReleaseDate'],
                            "Type":             release['Type'],
                            "ReleaseCountry":   release['ReleaseCountry'],
                            "ReleaseFormat":    release['ReleaseFormat']
                        }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            for track in release['Tracks']:

                cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title'])

                controlValueDict = {"TrackID":      track['id'],
                                    "ReleaseID":    release['ReleaseID']}

                newValueDict = {"ArtistID":         release['ArtistID'],
                                "ArtistName":       release['ArtistName'],
                                "AlbumTitle":       release['AlbumTitle'],
                                "AlbumID":          release['AlbumID'],
                                "AlbumASIN":        release['AlbumASIN'],
                                "TrackTitle":       track['title'],
                                "TrackDuration":    track['duration'],
                                "TrackNumber":      track['number'],
                                "CleanName":        cleanname
                            }
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
    cron=False):

    if cron and not headphones.CONFIG.LIBRARYSCAN:
        return

    if not dir:
        if not headphones.CONFIG.MUSIC_DIR:
            return
        else:
            dir = headphones.CONFIG.MUSIC_DIR

    # If we're appending a dir, it's coming from the post processor which is
    # already bytestring
    if not append:
        dir = dir.encode(headphones.SYS_ENCODING)

    if not os.path.isdir(dir):
        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace'))
        return

    myDB = db.DBConnection()
    new_artists = []

    logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))

    if not append:
        # Clean up bad filepaths
        tracks = myDB.select('SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL')

        for track in tracks:
            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING)
            if not os.path.isfile(encoded_track_string):
                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])
                myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])

        del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have')

        for track in del_have_tracks:
            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
            if not os.path.isfile(encoded_track_string):
                if track['ArtistName']:
                    # Make sure deleted files get accounted for when updating artist track counts
                    new_artists.append(track['ArtistName'])
                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
                logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace'))

    bitrates = []
    song_list = []
    latest_subdirectory = []

    new_song_count = 0
    file_count = 0

    for r, d, f in helpers.walk_directory(dir):
        # Filter paths based on config. Note that these methods work directly
        # on the inputs
        helpers.path_filter_patterns(d, headphones.CONFIG.IGNORED_FOLDERS, r)
        helpers.path_filter_patterns(f, headphones.CONFIG.IGNORED_FILES, r)

        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
                subdirectory = r.replace(dir, '')
                latest_subdirectory.append(subdirectory)

                if file_count == 0 and r.replace(dir, '') != '':
                    logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
                elif latest_subdirectory[file_count] != latest_subdirectory[file_count - 1] and file_count != 0:
                    logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace')))

                song = os.path.join(r, files)

                # We need the unicode path to use for logging, inserting into database
                unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace')

                # Try to read the metadata
                try:
                    f = MediaFile(song)
                except (FileTypeError, UnreadableFileError):
                    logger.warning("Cannot read media file '%s', skipping. It may be corrupted or not a media file.", unicode_song_path)
                    continue
                except IOError:
                    logger.warning("Cannnot read media file '%s', skipping. Does the file exists?", unicode_song_path)
                    continue

                # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)

                # Use the album artist over the artist if available
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    f_artist = None

                # Add the song to our song list -
                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)

                if f_artist and f.album and f.title:
                    CleanName = helpers.cleanName(f_artist + ' ' + f.album + ' ' + f.title)
                else:
                    CleanName = None

                controlValueDict = {'Location': unicode_song_path}

                newValueDict = {'TrackID': f.mb_trackid,
                              #'ReleaseID' : f.mb_albumid,
                              'ArtistName': f_artist,
                              'AlbumTitle': f.album,
                              'TrackNumber': f.track,
                              'TrackLength': f.length,
                              'Genre': f.genre,
                              'Date': f.date,
                              'TrackTitle': f.title,
                              'BitRate': f.bitrate,
                              'Format': f.format,
                              'CleanName': CleanName
                              }

                #song_list.append(song_dict)
                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone()
                #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
                if not check_exist_song:
                    #This is a new track
                    if f_artist:
                        new_artists.append(f_artist)
                    myDB.upsert("have", newValueDict, controlValueDict)
                    new_song_count += 1
                else:
                    if check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
                        #Important track metadata has been modified, need to run matcher again
                        if f_artist and f_artist != check_exist_song['ArtistName']:
                            new_artists.append(f_artist)
                        elif f_artist and f_artist == check_exist_song['ArtistName'] and check_exist_song['Matched'] != "Ignored":
                            new_artists.append(f_artist)
                        else:
                            continue

                        newValueDict['Matched'] = None
                        myDB.upsert("have", newValueDict, controlValueDict)
                        myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path])
                        myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path])
                        new_song_count += 1
                    else:
                        #This track information hasn't changed
                        if f_artist and check_exist_song['Matched'] != "Ignored":
                            new_artists.append(f_artist)

                file_count += 1

    # Now we start track matching
    logger.info("%s new/modified songs found and added to the database" % new_song_count)
    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"])
    total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0]
    logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")

    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches

    ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])

    # We'll use this to give a % completion, just because the track matching might take a while
    song_count = 0
    latest_artist = []

    for song in song_list:

        latest_artist.append(song['ArtistName'])
        if song_count == 0:
            logger.info("Now matching songs by %s" % song['ArtistName'])
        elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0:
            logger.info("Now matching songs by %s" % song['ArtistName'])

        song_count += 1
        completion_percentage = float(song_count) / total_number_of_songs * 100

        if completion_percentage % 10 == 0:
            logger.info("Track matching is " + str(completion_percentage) + "% complete")

        #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED.  WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT
        #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME.  ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION,
        #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
        #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.

        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:

            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
            have_updated = False
            if track:
                controlValueDict = {'ArtistName': track['ArtistName'],
                                     'AlbumTitle': track['AlbumTitle'],
                                     'TrackTitle': track['TrackTitle']}
                newValueDict = {'Location': song['Location'],
                                 'BitRate': song['BitRate'],
                                 'Format': song['Format']}
                myDB.upsert("tracks", newValueDict, controlValueDict)

                controlValueDict2 = {'Location': song['Location']}
                newValueDict2 = {'Matched': track['AlbumID']}
                myDB.upsert("have", newValueDict2, controlValueDict2)
                have_updated = True
            else:
                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
                if track:
                    controlValueDict = {'CleanName': track['CleanName']}
                    newValueDict = {'Location': song['Location'],
                                     'BitRate': song['BitRate'],
                                     'Format': song['Format']}
                    myDB.upsert("tracks", newValueDict, controlValueDict)

                    controlValueDict2 = {'Location': song['Location']}
                    newValueDict2 = {'Matched': track['AlbumID']}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
                    have_updated = True
                else:
                    controlValueDict2 = {'Location': song['Location']}
                    newValueDict2 = {'Matched': "Failed"}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
                    have_updated = True

            alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
            if alltrack:
                controlValueDict = {'ArtistName': alltrack['ArtistName'],
                                     'AlbumTitle': alltrack['AlbumTitle'],
                                     'TrackTitle': alltrack['TrackTitle']}
                newValueDict = {'Location': song['Location'],
                                 'BitRate': song['BitRate'],
                                 'Format': song['Format']}
                myDB.upsert("alltracks", newValueDict, controlValueDict)

                controlValueDict2 = {'Location': song['Location']}
                newValueDict2 = {'Matched': alltrack['AlbumID']}
                myDB.upsert("have", newValueDict2, controlValueDict2)
            else:
                alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
                if alltrack:
                    controlValueDict = {'CleanName': alltrack['CleanName']}
                    newValueDict = {'Location': song['Location'],
                                     'BitRate': song['BitRate'],
                                     'Format': song['Format']}
                    myDB.upsert("alltracks", newValueDict, controlValueDict)

                    controlValueDict2 = {'Location': song['Location']}
                    newValueDict2 = {'Matched': alltrack['AlbumID']}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
                else:
                    # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks
                    if not have_updated:
                        controlValueDict2 = {'Location': song['Location']}
                        newValueDict2 = {'Matched': "Failed"}
                        myDB.upsert("have", newValueDict2, controlValueDict2)

        else:
            controlValueDict2 = {'Location': song['Location']}
            newValueDict2 = {'Matched': "Failed"}
            myDB.upsert("have", newValueDict2, controlValueDict2)

        #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])

    logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))

    if not append:
        logger.info('Updating scanned artist track counts')

        # Clean up the new artist list
        unique_artists = {}.fromkeys(new_artists).keys()
        current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')

        #There was a bug where artists with special characters (-,') would show up in new artists.
        artist_list = [
            x for x in unique_artists
            if helpers.cleanName(x).lower() not in [
                helpers.cleanName(y[0]).lower()
                for y in current_artists
            ]
        ]
        artists_checked = [
            x for x in unique_artists
            if helpers.cleanName(x).lower() in [
                helpers.cleanName(y[0]).lower()
                for y in current_artists
            ]
        ]

        # Update track counts
        for artist in artists_checked:
            # Have tracks are selected from tracks table and not all tracks because of duplicates
            # We update the track count upon an album switch to compliment this
            havetracks = (
                len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist]))
                + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist]))
            )
            # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
            # (can fix by getting rid of second len statement)
            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist])

        logger.info('Found %i new artists' % len(artist_list))

        if artist_list:
            if headphones.CONFIG.AUTO_ADD_ARTISTS:
                logger.info('Importing %i new artists' % len(artist_list))
                importer.artistlist_to_mbids(artist_list)
            else:
                logger.info('To add these artists, go to Manage->Manage New Artists')
                #myDB.action('DELETE from newartists')
                for artist in artist_list:
                    myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist])

        if headphones.CONFIG.DETECT_BITRATE:
            headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000

    else:
        # If we're appending a new album to the database, update the artists total track counts
        logger.info('Updating artist track counts')

        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName]))
        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])

    if not append:
        update_album_status()
        lastfm.getSimilar()

    logger.info('Library scan complete')
Exemple #10
0
def libraryScan(dir=None):

    if not dir:
        dir = headphones.MUSIC_DIR
        
    dir = dir.encode(headphones.SYS_ENCODING)
        
    if not os.path.isdir(dir):
        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING))
        return

    myDB = db.DBConnection()
    
    # Clean up bad filepaths
    tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL')
    
    for track in tracks:
        if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)):
            myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']])

    logger.info('Scanning music directory: %s' % dir)

    new_artists = []
    bitrates = []

    myDB.action('DELETE from have')
    
    for r,d,f in os.walk(dir):
        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):

                song = os.path.join(r, files)

                # We need the unicode path to use for logging, inserting into database
                unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace')

                # Try to read the metadata
                try:
                    f = MediaFile(song)

                except:
                    logger.error('Cannot read file: ' + unicode_song_path)
                    continue
                    
                # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)
                
                # Try to find a match based on artist/album/tracktitle
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    continue
                
                if f_artist and f.album and f.title:

                    track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone()
                        
                    if not track:
                        track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone()
                    
                    if track:
                        myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']])
                        continue        
                
                # Try to match on mbid if available and we couldn't find a match based on metadata
                if f.mb_trackid:

                    # Wondering if theres a better way to do this -> do one thing if the row exists,
                    # do something else if it doesn't
                    track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone()
        
                    if track:
                        myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']])
                        continue                
                
                # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
                new_artists.append(f_artist)
                
                # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database                
                myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, unicode_song_path, helpers.cleanName(f_artist+' '+f.album+' '+f.title), f.format])

    logger.info('Completed scanning directory: %s' % dir)
    
    # Clean up the new artist list
    unique_artists = {}.fromkeys(new_artists).keys()
    current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
    
    artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]]
    
    # Update track counts
    logger.info('Updating track counts')

    for artist in current_artists:
        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']]))
        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']])
        
    logger.info('Found %i new artists' % len(artist_list))

    if len(artist_list):
        if headphones.ADD_ARTISTS:
            logger.info('Importing %i new artists' % len(artist_list))
            importer.artistlist_to_mbids(artist_list)
        else:
            logger.info('To add these artists, go to Manage->Manage New Artists')
            myDB.action('DELETE from newartists')
            for artist in artist_list:
                myDB.action('INSERT into newartists VALUES (?)', [artist])
    
    if headphones.DETECT_BITRATE:
        headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
Exemple #11
0
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None):

    if not dir:
        dir = headphones.MUSIC_DIR
    
    # If we're appending a dir, it's coming from the post processor which is
    # already bytestring
    if not append:
        dir = dir.encode(headphones.SYS_ENCODING)
        
    if not os.path.isdir(dir):
        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace'))
        return

    myDB = db.DBConnection()
    
    if not append:
        # Clean up bad filepaths
        tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL')
    
        for track in tracks:
            if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)):
                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']])

        myDB.action('DELETE from have')

    logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))

    new_artists = []
    bitrates = []
    
    song_list = []
    
    for r,d,f in os.walk(dir):
        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):

                song = os.path.join(r, files)

                # We need the unicode path to use for logging, inserting into database
                unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace')

                # Try to read the metadata
                try:
                    f = MediaFile(song)

                except:
                    logger.error('Cannot read file: ' + unicode_song_path)
                    continue
                    
                # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)
                    
                # Use the album artist over the artist if available
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    f_artist = None
                    
                # Add the song to our song list - 
                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)

                song_dict = { 'TrackID' : f.mb_trackid,
                              'ReleaseID' : f.mb_albumid,
                              'ArtistName' : f_artist,
                              'AlbumTitle' : f.album,
                              'TrackNumber': f.track,
                              'TrackLength': f.length,
                              'Genre'      : f.genre,
                              'Date'       : f.date,
                              'TrackTitle' : f.title,
                              'BitRate'    : f.bitrate,
                              'Format'     : f.format,
                              'Location'   : unicode_song_path }
                              
                song_list.append(song_dict)

    # Now we start track matching
    total_number_of_songs = len(song_list)
    logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
    
    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
    
    song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
    
    # We'll use this to give a % completion, just because the track matching might take a while
    song_count = 0
    
    for song in song_list:
        
        song_count += 1
        completion_percentage = float(song_count)/total_number_of_songs * 100
        
        if completion_percentage%10 == 0:
            logger.info("Track matching is " + str(completion_percentage) + "% complete")
        
        # If the track has a trackid & releaseid (beets: albumid) that the most surefire way
        # of identifying a track to a specific release so we'll use that first
        if song['TrackID'] and song['ReleaseID']:

            # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet
            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
            
            # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table
            if not track:
                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
    
            if track:
                # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo
                controlValueDict = { 'TrackID'   : track['TrackID'],
                                     'ReleaseID' : track['ReleaseID'] }
                
                # Insert it into the Headphones hybrid release (ReleaseID == AlbumID)                   
                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
                                           'ReleaseID' : track['AlbumID'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }
                                 
                # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict
                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)
                
                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
                
                # Matched. Move on to the next one:
                continue
    
        # If we can't find it with TrackID & ReleaseID, next most specific will be 
        # releaseid + tracktitle, although perhaps less reliable due to a higher 
        # likelihood of variations in the song title (e.g. feat. artists)
        if song['ReleaseID'] and song['TrackTitle']:
    
            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
    
            if not track:
                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
                
            if track:
                # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table)
                # So use both TrackID & ReleaseID as the control values
                controlValueDict = { 'TrackID'   : track['TrackID'],
                                     'ReleaseID' : track['ReleaseID'] }
                                     
                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
                                           'ReleaseID' : track['AlbumID'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }
                                 
                # Update both tables here as well
                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)
                
                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
                
                # Done
                continue
                
        # Next most specific will be the opposite: a TrackID and an AlbumTitle
        # TrackIDs span multiple releases so if something is on an official album
        # and a compilation, for example, this will match it to the right one
        # However - there may be multiple matches here
        if song['TrackID'] and song['AlbumTitle']:
    
            # Even though there might be multiple matches, we just need to grab one to confirm a match
            track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
    
            if not track:
                track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
                
            if track:
                # Don't need the hybridControlValueDict here since ReleaseID is not unique
                controlValueDict = { 'TrackID'   : track['TrackID'],
                                     'AlbumTitle' : track['AlbumTitle'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }

                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)

                continue   
        
        # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably 
        # even more unreliable than the previous queries, and might span multiple releases)
        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
            
            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
    
            if not track:
                track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
                
            if track:
                controlValueDict = { 'ArtistName' : track['ArtistName'],
                                     'AlbumTitle' : track['AlbumTitle'],
                                     'TrackTitle' : track['TrackTitle'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }

                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)

                continue
        
        # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc)
        # This is more reliable than the former but requires some string manipulation so we'll do it only
        # if we can't find a match with the original data
        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
            
            CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
            
            track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
            
            if not track:
                track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
    
            if track:
                controlValueDict = { 'CleanName' : track['CleanName'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }

                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)

                continue     
        
        # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable
        # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track
        # with more specific information exists in the library, it'll overwrite these values
        if song['TrackID']:
    
            track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone()
            
            if not track:
                track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone()
    
            if track:
                controlValueDict = { 'TrackID' : track['TrackID'] }
                                     
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }

                myDB.upsert("alltracks", newValueDict, controlValueDict)
                myDB.upsert("tracks", newValueDict, controlValueDict)

                continue          
        
        # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
        if song['ArtistName']:
            new_artists.append(song['ArtistName'])
        else:
            continue
        
        # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database                
        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
            CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
        else:
            continue
        
        myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])

    logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
    
    
    if not append:
        # Clean up the new artist list
        unique_artists = {}.fromkeys(new_artists).keys()
        current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
        
        artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]]
        
        # Update track counts
        logger.info('Updating current artist track counts')
    
        for artist in current_artists:
            # Have tracks are selected from tracks table and not all tracks because of duplicates
            # We update the track count upon an album switch to compliment this
            havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']]))
            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']])
            
        logger.info('Found %i new artists' % len(artist_list))
    
        if len(artist_list):
            if headphones.ADD_ARTISTS:
                logger.info('Importing %i new artists' % len(artist_list))
                importer.artistlist_to_mbids(artist_list)
            else:
                logger.info('To add these artists, go to Manage->Manage New Artists')
                myDB.action('DELETE from newartists')
                for artist in artist_list:
                    myDB.action('INSERT into newartists VALUES (?)', [artist])
        
        if headphones.DETECT_BITRATE:
            headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
            
    else:
        # If we're appending a new album to the database, update the artists total track counts
        logger.info('Updating artist track counts')
        
        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName]))
        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
  def scrapeScan(self, dir=None):
    for r,d,f in os.walk(dir):
      for files in f:
        # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
        if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
          song = os.path.join(r, files)
          file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors='replace')

          # Try to read the metadata
          try:
            f = MediaFile(song)
          except Exception, e:
            logger.error('Got Exception: ' + str(e))
            logger.error('Cannot read file: ' + file)
            continue

          # Grab the bitrates for the auto detect bit rate option
          if f.bitrate:
            bitrates.append(f.bitrate)

          # Try to find a match based on artist/album/tracktitle
          if f.albumartist:
            f_artist = f.albumartist
          elif f.artist:
            f_artist = f.artist
          else:
            continue

          if f_artist and f.album and f.title:
            track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone()

            if not track:
              track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone()

            if track:
              myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']])
              continue

          # Try to match on mbid if available and we couldn't find a match based on metadata
          if f.mb_trackid:
            # Wondering if theres a better way to do this -> do one thing if the row exists,
            # do something else if it doesn't
            track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone()

            if track:
              myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']])
              continue

          # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
          new_artists.append(f_artist)

          # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database        
          myDB.action('INSERT INTO have VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist+' '+f.album+' '+f.title)])
Exemple #13
0
def addArtisttoDB(artistid, extrasonly=False):

    # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
    from headphones import cache

    # Can't add various artists - throws an error from MB
    if artistid == various_artists_mbid:
        logger.warn('Cannot import Various Artists.')
        return

    # We'll use this to see if we should update the 'LastUpdated' time stamp
    errors = False

    myDB = db.DBConnection()

    # Delete from blacklist if it's on there
    myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])

    # We need the current minimal info in the database instantly
    # so we don't throw a 500 error when we redirect to the artistPage

    controlValueDict = {"ArtistID": artistid}

    # Don't replace a known artist name with an "Artist ID" placeholder

    dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?',
                           [artistid]).fetchone()

    # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
    if not dbartist:
        newValueDict = {
            "ArtistName": "Artist ID: %s" % (artistid),
            "Status": "Loading",
            "IncludeExtras": headphones.INCLUDE_EXTRAS,
            "Extras": headphones.EXTRAS
        }
    else:
        newValueDict = {"Status": "Loading"}

    myDB.upsert("artists", newValueDict, controlValueDict)

    artist = mb.getArtist(artistid, extrasonly)

    if not artist:
        logger.warn("Error fetching artist info. ID: " + artistid)
        if dbartist is None:
            newValueDict = {
                "ArtistName":
                "Fetch failed, try refreshing. (%s)" % (artistid),
                "Status": "Active"
            }
        else:
            newValueDict = {"Status": "Active"}
        myDB.upsert("artists", newValueDict, controlValueDict)
        return

    if artist['artist_name'].startswith('The '):
        sortname = artist['artist_name'][4:]
    else:
        sortname = artist['artist_name']

    logger.info(u"Now adding/updating: " + artist['artist_name'])
    controlValueDict = {"ArtistID": artistid}
    newValueDict = {
        "ArtistName": artist['artist_name'],
        "ArtistSortName": sortname,
        "DateAdded": helpers.today(),
        "Status": "Loading"
    }

    myDB.upsert("artists", newValueDict, controlValueDict)

    for rg in artist['releasegroups']:

        logger.info("Now adding/updating: " + rg['title'])

        rgid = rg['id']

        # check if the album already exists
        rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?",
                                [rg['id']]).fetchone()

        try:
            releaselist = mb.getReleaseGroup(rgid)
        except Exception, e:
            logger.info(
                'Unable to get release information for %s - there may not be any official releases in this release group'
                % rg['title'])
            continue

        if not releaselist:
            errors = True
            continue

        # This will be used later to build a hybrid release
        fullreleaselist = []

        for release in releaselist:
            # What we're doing here now is first updating the allalbums & alltracks table to the most
            # current info, then moving the appropriate release into the album table and its associated
            # tracks into the tracks table

            releaseid = release['id']

            try:
                releasedict = mb.getRelease(releaseid,
                                            include_artist_info=False)
            except Exception, e:
                errors = True
                logger.info('Unable to get release information for %s: %s' %
                            (release['id'], e))
                continue

            if not releasedict:
                errors = True
                continue

            controlValueDict = {"ReleaseID": release['id']}

            newValueDict = {
                "ArtistID": artistid,
                "ArtistName": artist['artist_name'],
                "AlbumTitle": rg['title'],
                "AlbumID": rg['id'],
                "AlbumASIN": releasedict['asin'],
                "ReleaseDate": releasedict['date'],
                "Type": rg['type'],
                "ReleaseCountry": releasedict['country'],
                "ReleaseFormat": releasedict['format']
            }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            # Build the dictionary for the fullreleaselist
            newValueDict['ReleaseID'] = release['id']
            newValueDict['Tracks'] = releasedict['tracks']
            fullreleaselist.append(newValueDict)

            for track in releasedict['tracks']:

                cleanname = helpers.cleanName(artist['artist_name'] + ' ' +
                                              rg['title'] + ' ' +
                                              track['title'])

                controlValueDict = {
                    "TrackID": track['id'],
                    "ReleaseID": release['id']
                }

                newValueDict = {
                    "ArtistID": artistid,
                    "ArtistName": artist['artist_name'],
                    "AlbumTitle": rg['title'],
                    "AlbumASIN": releasedict['asin'],
                    "AlbumID": rg['id'],
                    "TrackTitle": track['title'],
                    "TrackDuration": track['duration'],
                    "TrackNumber": track['number'],
                    "CleanName": cleanname
                }

                match = myDB.action(
                    'SELECT Location, BitRate, Format from have WHERE CleanName=?',
                    [cleanname]).fetchone()

                if not match:
                    match = myDB.action(
                        'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                        [artist['artist_name'], rg['title'], track['title']
                         ]).fetchone()
                if not match:
                    match = myDB.action(
                        'SELECT Location, BitRate, Format from have WHERE TrackID=?',
                        [track['id']]).fetchone()
                if match:
                    newValueDict['Location'] = match['Location']
                    newValueDict['BitRate'] = match['BitRate']
                    newValueDict['Format'] = match['Format']
                    myDB.action(
                        'UPDATE have SET Matched="True" WHERE Location=?',
                        [match['Location']])

                myDB.upsert("alltracks", newValueDict, controlValueDict)
                        "AlbumTitle":       release_dict['rg_title'],
                        "AlbumASIN":        release_dict['asin'],
                        "ReleaseDate":      release_dict['date'],
                        "DateAdded":        helpers.today(),
                        "Status":           'Wanted',
                        "Type":             release_dict['rg_type']
                        }
        
        myDB.upsert("albums", newValueDict, controlValueDict)

        #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB
        myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']])
        
        for track in release_dict['tracks']:
        
            cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title'])
            
            controlValueDict = {"TrackID":  track['id'],
                                "AlbumID":  rgid}
            newValueDict = {"ArtistID":     release_dict['artist_id'],
                        "ArtistName":       release_dict['artist_name'],
                        "AlbumTitle":       release_dict['rg_title'],
                        "AlbumASIN":        release_dict['asin'],
                        "TrackTitle":       track['title'],
                        "TrackDuration":    track['duration'],
                        "TrackNumber":      track['number'],
                        "CleanName":        cleanname
                        }
            
            match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
                        
Exemple #15
0
def addReleaseById(rid, rgid=None):

    myDB = db.DBConnection()

    # Create minimum info upfront if added from searchresults
    status = ''
    if rgid:
        dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid])
        if not dbalbum:
            status = 'Loading'
            controlValueDict = {"AlbumID":  rgid}
            newValueDict = {"AlbumTitle":   rgid,
                            "ArtistName":   status,
                            "Status":       status}
            myDB.upsert("albums", newValueDict, controlValueDict)
            time.sleep(1)

    rgid = None
    artistid = None
    release_dict = None
    results = myDB.select("SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid])
    for result in results:
        rgid = result['ReleaseGroupID']
        artistid = result['ArtistID']
        logger.debug("Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid)
    if not rgid:
        #didn't find it in the cache, get the information from MB
        logger.debug("Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID")
        try:
            release_dict = mb.getRelease(rid)
        except Exception as e:
            logger.info('Unable to get release information for Release %s: %s', rid, e)
            if status == 'Loading':
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
            return
        if not release_dict:
            logger.info('Unable to get release information for Release %s: no dict', rid)
            if status == 'Loading':
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
            return

        rgid = release_dict['rgid']
        artistid = release_dict['artist_id']

    #we don't want to make more calls to MB here unless we have to, could be happening quite a lot
    rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid])

    #make sure the artist exists since I don't know what happens later if it doesn't
    artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid])

    if not artist_exists and release_dict:
        if release_dict['artist_name'].startswith('The '):
            sortname = release_dict['artist_name'][4:]
        else:
            sortname = release_dict['artist_name']

        logger.info(u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
        controlValueDict = {"ArtistID":     release_dict['artist_id']}
        newValueDict = {"ArtistName":       release_dict['artist_name'],
                        "ArtistSortName":   sortname,
                        "DateAdded":        helpers.today(),
                        "Status":           "Paused"}

        if headphones.INCLUDE_EXTRAS:
            newValueDict['IncludeExtras'] = 1
            newValueDict['Extras'] = headphones.EXTRAS

        myDB.upsert("artists", newValueDict, controlValueDict)

    elif not artist_exists and not release_dict:
        logger.error("Artist does not exist in the database and did not get a valid response from MB. Skipping release.")
        if status == 'Loading':
            myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
        return

    if not rg_exists and release_dict or status == 'Loading' and release_dict:  #it should never be the case that we have an rg and not the artist
                                                                                #but if it is this will fail
        logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
        controlValueDict = {"AlbumID":  rgid}
        if status != 'Loading':
            status = 'Wanted'

        newValueDict = {"ArtistID":         release_dict['artist_id'],
                        "ReleaseID":        rgid,
                        "ArtistName":       release_dict['artist_name'],
                        "AlbumTitle":       release_dict['rg_title'],
                        "AlbumASIN":        release_dict['asin'],
                        "ReleaseDate":      release_dict['date'],
                        "DateAdded":        helpers.today(),
                        "Status":           status,
                        "Type":             release_dict['rg_type'],
                        "ReleaseID":        rid
                        }

        myDB.upsert("albums", newValueDict, controlValueDict)

        #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB
        myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']])

        for track in release_dict['tracks']:
            cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title'])

            controlValueDict = {"TrackID":  track['id'],
                                "AlbumID":  rgid}
            newValueDict = {"ArtistID":     release_dict['artist_id'],
                        "ArtistName":       release_dict['artist_name'],
                        "AlbumTitle":       release_dict['rg_title'],
                        "AlbumASIN":        release_dict['asin'],
                        "TrackTitle":       track['title'],
                        "TrackDuration":    track['duration'],
                        "TrackNumber":      track['number'],
                        "CleanName":        cleanname
                        }

            match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()

            if not match:
                match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone()

            #if not match:
                #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()

            if match:
                newValueDict['Location'] = match['Location']
                newValueDict['BitRate'] = match['BitRate']
                newValueDict['Format'] = match['Format']
                #myDB.action('DELETE from have WHERE Location=?', [match['Location']])

            myDB.upsert("tracks", newValueDict, controlValueDict)

        # Reset status
        if status == 'Loading':
            controlValueDict = {"AlbumID":  rgid}
            newValueDict = {"Status":   "Wanted"}
            myDB.upsert("albums", newValueDict, controlValueDict)

        # Start a search for the album
        import searcher
        searcher.searchforalbum(rgid, False)
    elif not rg_exists and not release_dict:
        logger.error("ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.")
        if status == 'Loading':
            myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
        return
    else:
        logger.info('Release ' + str(rid) + " already exists in the database!")
def addArtisttoDB(artistid, extrasonly=False):
    
    # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
    from headphones import cache
    
    # Can't add various artists - throws an error from MB
    if artistid in blacklisted_special_artists:
        logger.warn('Cannot import blocked special purpose artist with id' + artistid)
        return
        
    # We'll use this to see if we should update the 'LastUpdated' time stamp
    errors = False
        
    myDB = db.DBConnection()
    
    # Delete from blacklist if it's on there
    myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])

    # We need the current minimal info in the database instantly
    # so we don't throw a 500 error when we redirect to the artistPage

    controlValueDict = {"ArtistID":     artistid}

    # Don't replace a known artist name with an "Artist ID" placeholder

    dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone()
    
    # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
    if not dbartist:
        newValueDict = {"ArtistName":   "Artist ID: %s" % (artistid),
                        "Status":       "Loading",
                        "IncludeExtras": headphones.INCLUDE_EXTRAS,
                        "Extras":        headphones.EXTRAS }
    else:
        newValueDict = {"Status":   "Loading"}

    myDB.upsert("artists", newValueDict, controlValueDict)
        
    artist = mb.getArtist(artistid, extrasonly)
    
    if artist and artist.get('artist_name') in blacklisted_special_artist_names:
        logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name'))
        myDB.action('DELETE from artists WHERE ArtistID=?', [artistid])
        #in case it's already in the db
        myDB.action('DELETE from albums WHERE ArtistID=?', [artistid])
        myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid])
        return

    if not artist:
        logger.warn("Error fetching artist info. ID: " + artistid)
        if dbartist is None:
            newValueDict = {"ArtistName":   "Fetch failed, try refreshing. (%s)" % (artistid),
                    "Status":   "Active"}
        else:
            newValueDict = {"Status":   "Active"}
        myDB.upsert("artists", newValueDict, controlValueDict)
        return
    
    if artist['artist_name'].startswith('The '):
        sortname = artist['artist_name'][4:]
    else:
        sortname = artist['artist_name']
        

    logger.info(u"Now adding/updating: " + artist['artist_name'])
    controlValueDict = {"ArtistID":     artistid}
    newValueDict = {"ArtistName":       artist['artist_name'],
                    "ArtistSortName":   sortname,
                    "DateAdded":        helpers.today(),
                    "Status":           "Loading"}
    
    myDB.upsert("artists", newValueDict, controlValueDict)

    # See if we need to grab extras. Artist specific extras take precedence over global option
    # Global options are set when adding a new artist
    myDB = db.DBConnection()
    
    try:
        db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone()
        includeExtras = db_artist['IncludeExtras']
    except IndexError:
        includeExtras = False  

    for rg in artist['releasegroups']:
        
        logger.info("Now adding/updating: " + rg['title'])
        
        rgid = rg['id']
        
        # check if the album already exists
        rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
                    
        releases = mb.get_all_releases(rgid,includeExtras)
        if releases == []:
            logger.info('No official releases in release group %s' % rg['title'])
            continue
        if not releases:
            errors = True
            logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title'])
            continue
        
        # This will be used later to build a hybrid release     
        fullreleaselist = []

        for release in releases:
        # What we're doing here now is first updating the allalbums & alltracks table to the most
        # current info, then moving the appropriate release into the album table and its associated
        # tracks into the tracks table
            controlValueDict = {"ReleaseID" : release['ReleaseID']}

            newValueDict = {"ArtistID":         release['ArtistID'],
                            "ArtistName":       release['ArtistName'],
                            "AlbumTitle":       release['AlbumTitle'],
                            "AlbumID":          release['AlbumID'],
                            "AlbumASIN":        release['AlbumASIN'],
                            "ReleaseDate":      release['ReleaseDate'],
                            "Type":             release['Type'],
                            "ReleaseCountry":   release['ReleaseCountry'],
                            "ReleaseFormat":    release['ReleaseFormat']
                        }

            myDB.upsert("allalbums", newValueDict, controlValueDict)
            
            # Build the dictionary for the fullreleaselist
            newValueDict['ReleaseID'] = release['ReleaseID']
            newValueDict['Tracks'] = release['Tracks']
            fullreleaselist.append(newValueDict)
            
            for track in release['Tracks']:

                cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
        
                controlValueDict = {"TrackID":      track['id'],
                                    "ReleaseID":    release['ReleaseID']}

                newValueDict = {"ArtistID":         release['ArtistID'],
                                "ArtistName":       release['ArtistName'],
                                "AlbumTitle":       release['AlbumTitle'],
                                "AlbumID":          release['AlbumID'],
                                "AlbumASIN":        release['AlbumASIN'],
                                "TrackTitle":       track['title'],
                                "TrackDuration":    track['duration'],
                                "TrackNumber":      track['number'],
                                "CleanName":        cleanname
                            }
                            
                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
            
                if not match:
                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
                if not match:
                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
                if match:
                    newValueDict['Location'] = match['Location']
                    newValueDict['BitRate'] = match['BitRate']
                    newValueDict['Format'] = match['Format']
                    myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                                
                myDB.upsert("alltracks", newValueDict, controlValueDict)

        # Basically just do the same thing again for the hybrid release
        # This may end up being called with an empty fullreleaselist
        try:
            hybridrelease = getHybridRelease(fullreleaselist)
        except Exception, e:
            errors = True
            logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e))
            continue
        
        # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
        # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
        # The hybrid won't have a country or a format
        controlValueDict = {"ReleaseID":  rg['id']}

        newValueDict = {"ArtistID":         artistid,
                        "ArtistName":       artist['artist_name'],
                        "AlbumTitle":       rg['title'],
                        "AlbumID":          rg['id'],
                        "AlbumASIN":        hybridrelease['AlbumASIN'],
                        "ReleaseDate":      hybridrelease['ReleaseDate'],
                        "Type":             rg['type']
                    }
                    
        myDB.upsert("allalbums", newValueDict, controlValueDict)
        
        for track in hybridrelease['Tracks']:

            cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
    
            controlValueDict = {"TrackID":      track['id'],
                                "ReleaseID":    rg['id']}

            newValueDict = {"ArtistID":         artistid,
                            "ArtistName":       artist['artist_name'],
                            "AlbumTitle":       rg['title'],
                            "AlbumASIN":        hybridrelease['AlbumASIN'],
                            "AlbumID":          rg['id'],
                            "TrackTitle":       track['title'],
                            "TrackDuration":    track['duration'],
                            "TrackNumber":      track['number'],
                            "CleanName":        cleanname
                        }
                        
            match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
        
            if not match:
                match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
            if not match:
                match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
            if match:
                newValueDict['Location'] = match['Location']
                newValueDict['BitRate'] = match['BitRate']
                newValueDict['Format'] = match['Format']
                myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                            
            myDB.upsert("alltracks", newValueDict, controlValueDict)
        
        # Delete matched tracks from the have table
        myDB.action('DELETE from have WHERE Matched="True"')
        
        # If there's no release in the main albums tables, add the default (hybrid)
        # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
        if not rg_exists:
            releaseid = rg['id']
        elif rg_exists and not rg_exists['ReleaseID']:
            # Need to do some importing here - to transition the old format of using the release group
            # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID
            # so we'll need to move over the locations, bitrates & formats from the tracks table to the new
            # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups
            logger.info("Copying current track information to alternate releases")
            tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall()
            for track in tracks:
                if track['Location']:
                    controlValueDict = {"TrackID":  track['TrackID']}
                    newValueDict = {"Location":     track['Location'],
                                    "BitRate":      track['BitRate'],
                                    "Format":       track['Format'],
                                    }
                    myDB.upsert("alltracks", newValueDict, controlValueDict)
            releaseid = rg['id']
        else:
            releaseid = rg_exists['ReleaseID']
        
        album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone()

        controlValueDict = {"AlbumID":  rg['id']}

        newValueDict = {"ArtistID":         album['ArtistID'],
                        "ArtistName":       album['ArtistName'],
                        "AlbumTitle":       album['AlbumTitle'],
                        "ReleaseID":        album['ReleaseID'],
                        "AlbumASIN":        album['AlbumASIN'],
                        "ReleaseDate":      album['ReleaseDate'],
                        "Type":             album['Type'],
                        "ReleaseCountry":   album['ReleaseCountry'],
                        "ReleaseFormat":    album['ReleaseFormat']
                    }
            
        if not rg_exists:
            
            today = helpers.today()
            
            newValueDict['DateAdded']= today
                            
            if headphones.AUTOWANT_ALL:
                newValueDict['Status'] = "Wanted"
            elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING:
                newValueDict['Status'] = "Wanted"
            # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
            # The first test just makes sure we have year-month-day
            elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING:
                newValueDict['Status'] = "Wanted"
            else:
                newValueDict['Status'] = "Skipped"
        
        myDB.upsert("albums", newValueDict, controlValueDict)

        myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']])
        tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()

        # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT
        total_track_count = len(tracks)
        
        for track in tracks:
        
            controlValueDict = {"TrackID":  track['TrackID'],
                                "AlbumID":  rg['id']}

            newValueDict = {"ArtistID":     track['ArtistID'],
                        "ArtistName":       track['ArtistName'],
                        "AlbumTitle":       track['AlbumTitle'],
                        "AlbumASIN":        track['AlbumASIN'],
                        "ReleaseID":        track['ReleaseID'],
                        "TrackTitle":       track['TrackTitle'],
                        "TrackDuration":    track['TrackDuration'],
                        "TrackNumber":      track['TrackNumber'],
                        "CleanName":        track['CleanName'],
                        "Location":         track['Location'],
                        "Format":           track['Format'],
                        "BitRate":          track['BitRate']
                        }
                        
            myDB.upsert("tracks", newValueDict, controlValueDict)

        # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
        have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']]))
        marked_as_downloaded = False
        
        if rg_exists:
            if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
                myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
                marked_as_downloaded = True
        else:
            if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
                myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
                marked_as_downloaded = True

        logger.info(u"Seeing if we need album art for " + rg['title'])
        cache.getThumb(AlbumID=rg['id'])
        
        #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected:
        if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL:    
            from headphones import searcher
            searcher.searchforalbum(albumid=rg['id'])
Exemple #17
0
def libraryScan(dir=None,
                append=False,
                ArtistID=None,
                ArtistName=None,
                cron=False,
                artistScan=False):

    if cron and not headphones.CONFIG.LIBRARYSCAN:
        return

    if not dir:
        if not headphones.CONFIG.MUSIC_DIR:
            return
        else:
            dir = headphones.CONFIG.MUSIC_DIR

    # If we're appending a dir, it's coming from the post processor which is
    # already bytestring
    if not append or artistScan:
        dir = dir.encode(headphones.SYS_ENCODING)

    if not os.path.isdir(dir):
        logger.warn('Cannot find directory: %s. Not scanning' %
                    dir.decode(headphones.SYS_ENCODING, 'replace'))
        return

    myDB = db.DBConnection()
    new_artists = []

    logger.info('Scanning music directory: %s' %
                dir.decode(headphones.SYS_ENCODING, 'replace'))

    if not append:
        # Clean up bad filepaths
        tracks = myDB.select(
            'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL'
        )

        for track in tracks:
            encoded_track_string = track['Location'].encode(
                headphones.SYS_ENCODING)
            if not os.path.isfile(encoded_track_string):
                myDB.action(
                    'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                    [None, None, None, track['Location']])
                myDB.action(
                    'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                    [None, None, None, track['Location']])

        del_have_tracks = myDB.select(
            'SELECT Location, Matched, ArtistName from have')

        for track in del_have_tracks:
            encoded_track_string = track['Location'].encode(
                headphones.SYS_ENCODING, 'replace')
            if not os.path.isfile(encoded_track_string):
                if track['ArtistName']:
                    # Make sure deleted files get accounted for when updating artist track counts
                    new_artists.append(track['ArtistName'])
                myDB.action('DELETE FROM have WHERE Location=?',
                            [track['Location']])
                logger.info(
                    'File %s removed from Headphones, as it is no longer on disk'
                    % encoded_track_string.decode(headphones.SYS_ENCODING,
                                                  'replace'))

    bitrates = []
    song_list = []
    latest_subdirectory = []

    new_song_count = 0
    file_count = 0

    for r, d, f in helpers.walk_directory(dir):
        # Filter paths based on config. Note that these methods work directly
        # on the inputs
        helpers.path_filter_patterns(d, headphones.CONFIG.IGNORED_FOLDERS, r)
        helpers.path_filter_patterns(f, headphones.CONFIG.IGNORED_FILES, r)

        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith('.' + x.lower())
                   for x in headphones.MEDIA_FORMATS):
                subdirectory = r.replace(dir, '')
                latest_subdirectory.append(subdirectory)

                if file_count == 0 and r.replace(dir, '') != '':
                    logger.info(
                        "[%s] Now scanning subdirectory %s" %
                        (dir.decode(headphones.SYS_ENCODING, 'replace'),
                         subdirectory.decode(headphones.SYS_ENCODING,
                                             'replace')))
                elif latest_subdirectory[file_count] != latest_subdirectory[
                        file_count - 1] and file_count != 0:
                    logger.info(
                        "[%s] Now scanning subdirectory %s" %
                        (dir.decode(headphones.SYS_ENCODING, 'replace'),
                         subdirectory.decode(headphones.SYS_ENCODING,
                                             'replace')))

                song = os.path.join(r, files)

                # We need the unicode path to use for logging, inserting into database
                unicode_song_path = song.decode(headphones.SYS_ENCODING,
                                                'replace')

                # Try to read the metadata
                try:
                    f = MediaFile(song)
                except (FileTypeError, UnreadableFileError):
                    logger.warning(
                        "Cannot read media file '%s', skipping. It may be corrupted or not a media file.",
                        unicode_song_path)
                    continue
                except IOError:
                    logger.warning(
                        "Cannnot read media file '%s', skipping. Does the file exists?",
                        unicode_song_path)
                    continue

                # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)

                # Use the album artist over the artist if available
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    f_artist = None

                # Add the song to our song list -
                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)

                if f_artist and f.album and f.title:
                    CleanName = helpers.cleanName(f_artist + ' ' + f.album +
                                                  ' ' + f.title)
                else:
                    CleanName = None

                controlValueDict = {'Location': unicode_song_path}

                newValueDict = {
                    'TrackID': f.mb_trackid,
                    #'ReleaseID' : f.mb_albumid,
                    'ArtistName': f_artist,
                    'AlbumTitle': f.album,
                    'TrackNumber': f.track,
                    'TrackLength': f.length,
                    'Genre': f.genre,
                    'Date': f.date,
                    'TrackTitle': f.title,
                    'BitRate': f.bitrate,
                    'Format': f.format,
                    'CleanName': CleanName
                }

                #song_list.append(song_dict)
                check_exist_song = myDB.action(
                    "SELECT * FROM have WHERE Location=?",
                    [unicode_song_path]).fetchone()
                #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
                if not check_exist_song:
                    #This is a new track
                    if f_artist:
                        new_artists.append(f_artist)
                    myDB.upsert("have", newValueDict, controlValueDict)
                    new_song_count += 1
                else:
                    if check_exist_song[
                            'ArtistName'] != f_artist or check_exist_song[
                                'AlbumTitle'] != f.album or check_exist_song[
                                    'TrackTitle'] != f.title:
                        #Important track metadata has been modified, need to run matcher again
                        if f_artist and f_artist != check_exist_song[
                                'ArtistName']:
                            new_artists.append(f_artist)
                        elif f_artist and f_artist == check_exist_song[
                                'ArtistName'] and check_exist_song[
                                    'Matched'] != "Ignored":
                            new_artists.append(f_artist)
                        else:
                            continue

                        newValueDict['Matched'] = None
                        myDB.upsert("have", newValueDict, controlValueDict)
                        myDB.action(
                            'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                            [None, None, None, unicode_song_path])
                        myDB.action(
                            'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                            [None, None, None, unicode_song_path])
                        new_song_count += 1
                    else:
                        #This track information hasn't changed
                        if f_artist and check_exist_song[
                                'Matched'] != "Ignored":
                            new_artists.append(f_artist)

                file_count += 1

    # Now we start track matching
    logger.info("%s new/modified songs found and added to the database" %
                new_song_count)
    song_list = myDB.action(
        "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
        [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"])
    total_number_of_songs = myDB.action(
        "SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
        [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0]
    logger.info("Found " + str(total_number_of_songs) +
                " new/modified tracks in: '" +
                dir.decode(headphones.SYS_ENCODING, 'replace') +
                "'. Matching tracks to the appropriate releases....")

    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches

    ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])

    # We'll use this to give a % completion, just because the track matching might take a while
    song_count = 0
    latest_artist = []

    for song in song_list:

        latest_artist.append(song['ArtistName'])
        if song_count == 0:
            logger.info("Now matching songs by %s" % song['ArtistName'])
        elif latest_artist[song_count] != latest_artist[song_count -
                                                        1] and song_count != 0:
            logger.info("Now matching songs by %s" % song['ArtistName'])

        song_count += 1
        completion_percentage = float(song_count) / total_number_of_songs * 100

        if completion_percentage % 10 == 0:
            logger.info("Track matching is " + str(completion_percentage) +
                        "% complete")

        #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED.  WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT
        #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME.  ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION,
        #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
        #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.

        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:

            track = myDB.action(
                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']
                 ]).fetchone()
            have_updated = False
            if track:
                controlValueDict = {
                    'ArtistName': track['ArtistName'],
                    'AlbumTitle': track['AlbumTitle'],
                    'TrackTitle': track['TrackTitle']
                }
                newValueDict = {
                    'Location': song['Location'],
                    'BitRate': song['BitRate'],
                    'Format': song['Format']
                }
                myDB.upsert("tracks", newValueDict, controlValueDict)

                controlValueDict2 = {'Location': song['Location']}
                newValueDict2 = {'Matched': track['AlbumID']}
                myDB.upsert("have", newValueDict2, controlValueDict2)
                have_updated = True
            else:
                track = myDB.action(
                    'SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?',
                    [song['CleanName']]).fetchone()
                if track:
                    controlValueDict = {'CleanName': track['CleanName']}
                    newValueDict = {
                        'Location': song['Location'],
                        'BitRate': song['BitRate'],
                        'Format': song['Format']
                    }
                    myDB.upsert("tracks", newValueDict, controlValueDict)

                    controlValueDict2 = {'Location': song['Location']}
                    newValueDict2 = {'Matched': track['AlbumID']}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
                    have_updated = True
                else:
                    controlValueDict2 = {'Location': song['Location']}
                    newValueDict2 = {'Matched': "Failed"}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
                    have_updated = True

            alltrack = myDB.action(
                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']
                 ]).fetchone()
            if alltrack:
                controlValueDict = {
                    'ArtistName': alltrack['ArtistName'],
                    'AlbumTitle': alltrack['AlbumTitle'],
                    'TrackTitle': alltrack['TrackTitle']
                }
                newValueDict = {
                    'Location': song['Location'],
                    'BitRate': song['BitRate'],
                    'Format': song['Format']
                }
                myDB.upsert("alltracks", newValueDict, controlValueDict)

                controlValueDict2 = {'Location': song['Location']}
                newValueDict2 = {'Matched': alltrack['AlbumID']}
                myDB.upsert("have", newValueDict2, controlValueDict2)
            else:
                alltrack = myDB.action(
                    'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?',
                    [song['CleanName']]).fetchone()
                if alltrack:
                    controlValueDict = {'CleanName': alltrack['CleanName']}
                    newValueDict = {
                        'Location': song['Location'],
                        'BitRate': song['BitRate'],
                        'Format': song['Format']
                    }
                    myDB.upsert("alltracks", newValueDict, controlValueDict)

                    controlValueDict2 = {'Location': song['Location']}
                    newValueDict2 = {'Matched': alltrack['AlbumID']}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
                else:
                    # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks
                    if not have_updated:
                        controlValueDict2 = {'Location': song['Location']}
                        newValueDict2 = {'Matched': "Failed"}
                        myDB.upsert("have", newValueDict2, controlValueDict2)

        else:
            controlValueDict2 = {'Location': song['Location']}
            newValueDict2 = {'Matched': "Failed"}
            myDB.upsert("have", newValueDict2, controlValueDict2)

        #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])

    logger.info('Completed matching tracks from directory: %s' %
                dir.decode(headphones.SYS_ENCODING, 'replace'))

    if not append or artistScan:
        logger.info('Updating scanned artist track counts')

        # Clean up the new artist list
        unique_artists = {}.fromkeys(new_artists).keys()
        current_artists = myDB.select(
            'SELECT ArtistName, ArtistID from artists')

        #There was a bug where artists with special characters (-,') would show up in new artists.
        artist_list = [
            x for x in unique_artists if helpers.cleanName(x).lower() not in
            [helpers.cleanName(y[0]).lower() for y in current_artists]
        ]
        artists_checked = [
            x for x in unique_artists if helpers.cleanName(x).lower() in
            [helpers.cleanName(y[0]).lower() for y in current_artists]
        ]

        # Update track counts
        for artist in artists_checked:
            # Have tracks are selected from tracks table and not all tracks because of duplicates
            # We update the track count upon an album switch to compliment this
            havetracks = (len(
                myDB.select(
                    'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
                    [artist])
            ) + len(
                myDB.select(
                    'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
                    [artist])))
            # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
            # (can fix by getting rid of second len statement)
            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?',
                        [havetracks, artist])

        logger.info('Found %i new artists' % len(artist_list))

        if artist_list:
            if headphones.CONFIG.AUTO_ADD_ARTISTS:
                logger.info('Importing %i new artists' % len(artist_list))
                importer.artistlist_to_mbids(artist_list)
            else:
                logger.info(
                    'To add these artists, go to Manage->Manage New Artists')
                #myDB.action('DELETE from newartists')
                for artist in artist_list:
                    myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)',
                                [artist])

        if headphones.CONFIG.DETECT_BITRATE and bitrates:
            headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len(
                bitrates) / 1000

    else:
        # If we're appending a new album to the database, update the artists total track counts
        logger.info('Updating artist track counts')

        havetracks = len(
            myDB.select(
                'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
                [ArtistID])
        ) + len(
            myDB.select(
                'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
                [ArtistName]))
        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?',
                    [havetracks, ArtistID])

    if not append:
        update_album_status()

    if not append and not artistScan:
        lastfm.getSimilar()

    logger.info('Library scan complete')
Exemple #18
0
def addArtisttoDB(artistid, extrasonly=False, forcefull=False):

    # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
    from headphones import cache

    # Can't add various artists - throws an error from MB
    if artistid in blacklisted_special_artists:
        logger.warn('Cannot import blocked special purpose artist with id' + artistid)
        return

    # We'll use this to see if we should update the 'LastUpdated' time stamp
    errors = False

    myDB = db.DBConnection()

    # Delete from blacklist if it's on there
    myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])

    # We need the current minimal info in the database instantly
    # so we don't throw a 500 error when we redirect to the artistPage
    controlValueDict = {"ArtistID": artistid}

    # Don't replace a known artist name with an "Artist ID" placeholder
    dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone()

    # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
    if not dbartist:
        newValueDict = {"ArtistName": "Artist ID: %s" % (artistid),
                        "Status": "Loading",
                        "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS,
                        "Extras": headphones.CONFIG.EXTRAS}
    else:
        newValueDict = {"Status": "Loading"}

    myDB.upsert("artists", newValueDict, controlValueDict)

    artist = mb.getArtist(artistid, extrasonly)

    if artist and artist.get('artist_name') in blacklisted_special_artist_names:
        logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name'))
        myDB.action('DELETE from artists WHERE ArtistID=?', [artistid])
        #in case it's already in the db
        myDB.action('DELETE from albums WHERE ArtistID=?', [artistid])
        myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid])
        return

    if not artist:
        logger.warn("Error fetching artist info. ID: " + artistid)
        if dbartist is None:
            newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid),
                    "Status": "Active"}
        else:
            newValueDict = {"Status": "Active"}
        myDB.upsert("artists", newValueDict, controlValueDict)
        return

    if artist['artist_name'].startswith('The '):
        sortname = artist['artist_name'][4:]
    else:
        sortname = artist['artist_name']

    logger.info(u"Now adding/updating: " + artist['artist_name'])
    controlValueDict = {"ArtistID": artistid}
    newValueDict = {"ArtistName": artist['artist_name'],
                    "ArtistSortName": sortname,
                    "DateAdded": helpers.today(),
                    "Status": "Loading"}

    myDB.upsert("artists", newValueDict, controlValueDict)

    # See if we need to grab extras. Artist specific extras take precedence
    # over global option. Global options are set when adding a new artist
    try:
        db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone()
        includeExtras = db_artist['IncludeExtras']
    except IndexError:
        includeExtras = False

    # Clean all references to release group in dB that are no longer referenced
    # from the musicbrainz refresh
    group_list = []
    force_repackage = 0

    # Don't nuke the database if there's a MusicBrainz error
    if len(artist['releasegroups']) != 0:
        for groups in artist['releasegroups']:
            group_list.append(groups['id'])
        if not extrasonly:
            remove_missing_groups_from_albums = myDB.select("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
        else:
            remove_missing_groups_from_albums = myDB.select('SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid])
        for items in remove_missing_groups_from_albums:
            if items['AlbumID'] not in group_list:
                # Remove all from albums/tracks that aren't in release groups
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']])
                myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']])
                myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']])
                myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']])
                myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']])
                logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (artist['artist_name'], items['AlbumID']))
                if not extrasonly:
                    force_repackage = 1
    else:
        if not extrasonly:
            logger.info("[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name'])

    # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
    album_searches = []

    for rg in artist['releasegroups']:
        al_title = rg['title']
        today = helpers.today()
        rgid = rg['id']
        skip_log = 0
        #Make a user configurable variable to skip update of albums with release dates older than this date (in days)
        pause_delta = headphones.CONFIG.MB_IGNORE_AGE

        rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()

        if not forcefull:
            new_release_group = False

            try:
                check_release_date = rg_exists['ReleaseDate']
            except TypeError:
                check_release_date = None
                new_release_group = True

            if new_release_group:
                logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title']))
                new_releases = mb.get_new_releases(rgid, includeExtras)

            else:
                if check_release_date is None or check_release_date == u"None":
                    logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
                    new_releases = mb.get_new_releases(rgid, includeExtras, True)
                else:
                    if len(check_release_date) == 10:
                        release_date = check_release_date
                    elif len(check_release_date) == 7:
                        release_date = check_release_date + "-31"
                    elif len(check_release_date) == 4:
                        release_date = check_release_date + "-12-31"
                    else:
                        release_date = today
                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
                        logger.info("[%s] Now updating: %s (Release Date <%s Days)", artist['artist_name'], rg['title'], pause_delta)
                        new_releases = mb.get_new_releases(rgid, includeExtras, True)
                    else:
                        logger.info("[%s] Skipping: %s (Release Date >%s Days)", artist['artist_name'], rg['title'], pause_delta)
                        skip_log = 1
                        new_releases = 0

            if force_repackage == 1:
                new_releases = -1
                logger.info('[%s] Forcing repackage of %s (Release Group Removed)', artist['artist_name'], al_title)
            else:
                new_releases = new_releases
        else:
            logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'], rg['title'])
            new_releases = mb.get_new_releases(rgid, includeExtras, forcefull)

        if new_releases != 0:
            # Dump existing hybrid release since we're repackaging/replacing it
            myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']])
            myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rg['id']])

            # This will be used later to build a hybrid release
            fullreleaselist = []
            # Search for releases within a release group
            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']])

            # Build the dictionary for the fullreleaselist
            for items in find_hybrid_releases:
                if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing
                    hybrid_release_id = items['ReleaseID']
                    newValueDict = {"ArtistID": items['ArtistID'],
                        "ArtistName": items['ArtistName'],
                        "AlbumTitle": items['AlbumTitle'],
                        "AlbumID": items['AlbumID'],
                        "AlbumASIN": items['AlbumASIN'],
                        "ReleaseDate": items['ReleaseDate'],
                        "Type": items['Type'],
                        "ReleaseCountry": items['ReleaseCountry'],
                        "ReleaseFormat": items['ReleaseFormat']
                    }
                    find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id])
                    totalTracks = 1
                    hybrid_track_array = []
                    for hybrid_tracks in find_hybrid_tracks:
                        hybrid_track_array.append({
                            'number': hybrid_tracks['TrackNumber'],
                            'title': hybrid_tracks['TrackTitle'],
                            'id': hybrid_tracks['TrackID'],
                            #'url':           hybrid_tracks['TrackURL'],
                            'duration': hybrid_tracks['TrackDuration']
                            })
                        totalTracks += 1
                    newValueDict['ReleaseID'] = hybrid_release_id
                    newValueDict['Tracks'] = hybrid_track_array
                    fullreleaselist.append(newValueDict)

            # Basically just do the same thing again for the hybrid release
            # This may end up being called with an empty fullreleaselist
            try:
                hybridrelease = getHybridRelease(fullreleaselist)
                logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title']))
            except Exception as e:
                errors = True
                logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'], rg['title'], e))
                continue

            # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
            # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
            # The hybrid won't have a country or a format
            controlValueDict = {"ReleaseID": rg['id']}

            newValueDict = {"ArtistID": artistid,
                            "ArtistName": artist['artist_name'],
                            "AlbumTitle": rg['title'],
                            "AlbumID": rg['id'],
                            "AlbumASIN": hybridrelease['AlbumASIN'],
                            "ReleaseDate": hybridrelease['ReleaseDate'],
                            "Type": rg['type']
                        }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            for track in hybridrelease['Tracks']:

                cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])

                controlValueDict = {"TrackID": track['id'],
                                    "ReleaseID": rg['id']}

                newValueDict = {"ArtistID": artistid,
                                "ArtistName": artist['artist_name'],
                                "AlbumTitle": rg['title'],
                                "AlbumASIN": hybridrelease['AlbumASIN'],
                                "AlbumID": rg['id'],
                                "TrackTitle": track['title'],
                                "TrackDuration": track['duration'],
                                "TrackNumber": track['number'],
                                "CleanName": cleanname
                            }

                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()

                if not match:
                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
                #if not match:
                    #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                if match:
                    newValueDict['Location'] = match['Location']
                    newValueDict['BitRate'] = match['BitRate']
                    newValueDict['Format'] = match['Format']
                    #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location']))

                myDB.upsert("alltracks", newValueDict, controlValueDict)

            # Delete matched tracks from the have table
            #myDB.action('DELETE from have WHERE Matched="True"')

            # If there's no release in the main albums tables, add the default (hybrid)
            # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
            # check if the album already exists
            if not rg_exists:
                releaseid = rg['id']
            else:
                releaseid = rg_exists['ReleaseID']
                if not releaseid:
                    releaseid = rg['id']

            album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone()

            controlValueDict = {"AlbumID": rg['id']}

            newValueDict = {"ArtistID": album['ArtistID'],
                            "ArtistName": album['ArtistName'],
                            "AlbumTitle": album['AlbumTitle'],
                            "ReleaseID": album['ReleaseID'],
                            "AlbumASIN": album['AlbumASIN'],
                            "ReleaseDate": album['ReleaseDate'],
                            "Type": album['Type'],
                            "ReleaseCountry": album['ReleaseCountry'],
                            "ReleaseFormat": album['ReleaseFormat']
                        }

            if rg_exists:
                newValueDict['DateAdded'] = rg_exists['DateAdded']
                newValueDict['Status'] = rg_exists['Status']

            else:
                today = helpers.today()

                newValueDict['DateAdded'] = today

                if headphones.CONFIG.AUTOWANT_ALL:
                    newValueDict['Status'] = "Wanted"
                elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING:
                    newValueDict['Status'] = "Wanted"
                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
                # The first test just makes sure we have year-month-day
                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
                    newValueDict['Status'] = "Wanted"
                else:
                    newValueDict['Status'] = "Skipped"

            myDB.upsert("albums", newValueDict, controlValueDict)

            tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()

            # This is used to see how many tracks you have from an album - to
            # mark it as downloaded. Default is 80%, can be set in config as
            # ALBUM_COMPLETION_PCT
            total_track_count = len(tracks)

            if total_track_count == 0:
                logger.warning("Total track count is zero for Release ID " +
                    "'%s', skipping.", releaseid)
                continue

            for track in tracks:
                controlValueDict = {"TrackID": track['TrackID'],
                                    "AlbumID": rg['id']}

                newValueDict = {"ArtistID": track['ArtistID'],
                            "ArtistName": track['ArtistName'],
                            "AlbumTitle": track['AlbumTitle'],
                            "AlbumASIN": track['AlbumASIN'],
                            "ReleaseID": track['ReleaseID'],
                            "TrackTitle": track['TrackTitle'],
                            "TrackDuration": track['TrackDuration'],
                            "TrackNumber": track['TrackNumber'],
                            "CleanName": track['CleanName'],
                            "Location": track['Location'],
                            "Format": track['Format'],
                            "BitRate": track['BitRate']
                            }

                myDB.upsert("tracks", newValueDict, controlValueDict)

            # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
            have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']]))
            marked_as_downloaded = False

            if rg_exists:
                if rg_exists['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
                    marked_as_downloaded = True
            else:
                if ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
                    marked_as_downloaded = True

            logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
            cache.getThumb(AlbumID=rg['id'])

            # Start a search for the album if it's new, hasn't been marked as
            # downloaded and autowant_all is selected. This search is deferred,
            # in case the search failes and the rest of the import will halt.
            if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL:
                album_searches.append(rg['id'])
        else:
            if skip_log == 0:
                logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title']))

    time.sleep(3)
    finalize_update(artistid, artist['artist_name'], errors)

    logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
    cache.getThumb(ArtistID=artistid)

    if errors:
        logger.info("[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name']))
    else:
        myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']])
        logger.info(u"Updating complete for: %s" % artist['artist_name'])

    # Start searching for newly added albums
    if album_searches:
        from headphones import searcher
        logger.info("Start searching for %d albums.", len(album_searches))

        for album_search in album_searches:
            searcher.searchforalbum(albumid=album_search)
Exemple #19
0
def addArtisttoDB(artistid, extrasonly=False, forcefull=False):
    
    # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
    from headphones import cache
    
    # Can't add various artists - throws an error from MB
    if artistid in blacklisted_special_artists:
        logger.warn('Cannot import blocked special purpose artist with id' + artistid)
        return
        
    # We'll use this to see if we should update the 'LastUpdated' time stamp
    errors = False
        
    myDB = db.DBConnection()
    
    # Delete from blacklist if it's on there
    myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])

    # We need the current minimal info in the database instantly
    # so we don't throw a 500 error when we redirect to the artistPage

    controlValueDict = {"ArtistID":     artistid}

    # Don't replace a known artist name with an "Artist ID" placeholder

    dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone()
    
    # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
    if not dbartist:
        newValueDict = {"ArtistName":   "Artist ID: %s" % (artistid),
                        "Status":       "Loading",
                        "IncludeExtras": headphones.INCLUDE_EXTRAS,
                        "Extras":        headphones.EXTRAS }
    else:
        newValueDict = {"Status":   "Loading"}

    myDB.upsert("artists", newValueDict, controlValueDict)
        
    artist = mb.getArtist(artistid, extrasonly)
    
    if artist and artist.get('artist_name') in blacklisted_special_artist_names:
        logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name'))
        myDB.action('DELETE from artists WHERE ArtistID=?', [artistid])
        #in case it's already in the db
        myDB.action('DELETE from albums WHERE ArtistID=?', [artistid])
        myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid])
        return

    if not artist:
        logger.warn("Error fetching artist info. ID: " + artistid)
        if dbartist is None:
            newValueDict = {"ArtistName":   "Fetch failed, try refreshing. (%s)" % (artistid),
                    "Status":   "Active"}
        else:
            newValueDict = {"Status":   "Active"}
        myDB.upsert("artists", newValueDict, controlValueDict)
        return
    
    if artist['artist_name'].startswith('The '):
        sortname = artist['artist_name'][4:]
    else:
        sortname = artist['artist_name']
        

    logger.info(u"Now adding/updating: " + artist['artist_name'])
    controlValueDict = {"ArtistID":     artistid}
    newValueDict = {"ArtistName":       artist['artist_name'],
                    "ArtistSortName":   sortname,
                    "DateAdded":        helpers.today(),
                    "Status":           "Loading"}
    
    myDB.upsert("artists", newValueDict, controlValueDict)

    # See if we need to grab extras. Artist specific extras take precedence over global option
    # Global options are set when adding a new artist
    myDB = db.DBConnection()
    
    try:
        db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone()
        includeExtras = db_artist['IncludeExtras']
    except IndexError:
        includeExtras = False

    #Clean all references to release group in dB that are no longer referenced in musicbrainz
    group_list = []
    force_repackage = 0
    #Don't nuke the database if there's a MusicBrainz error
    if len(artist['releasegroups']) != 0 and not extrasonly:
        for groups in artist['releasegroups']:
            group_list.append(groups['id'])
        remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
        for items in remove_missing_groups_from_albums:
            if items['AlbumID'] not in group_list:
                # Remove all from albums/tracks that aren't in release groups
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']])
                myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']])
                myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']])
                myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']])
                logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz" % (artist['artist_name'], items['AlbumID']))
                force_repackage = 1
    else:
        logger.info("[%s] Error pulling data from MusicBrainz:  Maintaining dB" % artist['artist_name'])
    
    # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks


    for rg in artist['releasegroups']:
        
        al_title = rg['title']
        today = helpers.today()
        rgid = rg['id']
        skip_log = 0
        #Make a user configurable variable to skip update of albums with release dates older than this date (in days)
        pause_delta = headphones.MB_IGNORE_AGE

        if not forcefull:
            check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone()
            if check_release_date:
                if check_release_date[0] is None:
                    logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
                    new_releases = mb.get_new_releases(rgid,includeExtras,True)          
                else:
                    if len(check_release_date[0]) == 10:
                        release_date = check_release_date[0]
                    elif len(check_release_date[0]) == 7:
                        release_date = check_release_date[0]+"-31"
                    elif len(check_release_date[0]) == 4:
                        release_date = check_release_date[0]+"-12-31"
                    else:
                        release_date = today
                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
                        logger.info("[%s] Now updating: %s (Release Date <%s Days) " % (artist['artist_name'], rg['title'], pause_delta))
                        new_releases = mb.get_new_releases(rgid,includeExtras,True)
                    else:
                        logger.info("[%s] Skipping: %s (Release Date >%s Days)" % (artist['artist_name'], rg['title'], pause_delta))
                        skip_log = 1
                        new_releases = 0
            else:
                logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title']))
                new_releases = mb.get_new_releases(rgid,includeExtras)

            if force_repackage == 1:
                new_releases = -1
                logger.info('[%s] Forcing repackage of %s (Release Group Removed)' % (artist['artist_name'], al_title))
            else:
                new_releases = new_releases
        else:
            logger.info("[%s] Now adding/updating: %s (Comprehensive Force)" % (artist['artist_name'], rg['title']))
            new_releases = mb.get_new_releases(rgid,includeExtras,forcefull)
        
        #What this does is adds new releases per artist to the allalbums + alltracks databases
        #new_releases = mb.get_new_releases(rgid,includeExtras)
        #print al_title
        #print new_releases

        if new_releases != 0:
            #Dump existing hybrid release since we're repackaging/replacing it
            myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']])
            # This will be used later to build a hybrid release     
            fullreleaselist = []
            #Search for releases within a release group
            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']])
            # Build the dictionary for the fullreleaselist
            for items in find_hybrid_releases:
                if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing
                    hybrid_release_id = items['ReleaseID']
                    newValueDict = {"ArtistID":         items['ArtistID'],
                        "ArtistName":       items['ArtistName'],
                        "AlbumTitle":       items['AlbumTitle'],
                        "AlbumID":          items['AlbumID'],
                        "AlbumASIN":        items['AlbumASIN'],
                        "ReleaseDate":      items['ReleaseDate'],
                        "Type":             items['Type'],
                        "ReleaseCountry":   items['ReleaseCountry'],
                        "ReleaseFormat":    items['ReleaseFormat']
                    }
                    find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id])
                    totalTracks = 1
                    hybrid_track_array = []
                    for hybrid_tracks in find_hybrid_tracks:
                        hybrid_track_array.append({
                            'number':        hybrid_tracks['TrackNumber'],
                            'title':         hybrid_tracks['TrackTitle'],
                            'id':            hybrid_tracks['TrackID'],
                            #'url':           hybrid_tracks['TrackURL'],
                            'duration':      hybrid_tracks['TrackDuration']
                            })
                        totalTracks += 1  
                    newValueDict['ReleaseID'] = hybrid_release_id
                    newValueDict['Tracks'] = hybrid_track_array
                    fullreleaselist.append(newValueDict)

                #print fullreleaselist

            # Basically just do the same thing again for the hybrid release
            # This may end up being called with an empty fullreleaselist
            try:
                hybridrelease = getHybridRelease(fullreleaselist)
                logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title']))
            except Exception, e:
                errors = True
                logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'],rg['title'],e))
                continue
            
            # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
            # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
            # The hybrid won't have a country or a format
            controlValueDict = {"ReleaseID":  rg['id']}

            newValueDict = {"ArtistID":         artistid,
                            "ArtistName":       artist['artist_name'],
                            "AlbumTitle":       rg['title'],
                            "AlbumID":          rg['id'],
                            "AlbumASIN":        hybridrelease['AlbumASIN'],
                            "ReleaseDate":      hybridrelease['ReleaseDate'],
                            "Type":             rg['type']
                        }
                        
            myDB.upsert("allalbums", newValueDict, controlValueDict)
            
            for track in hybridrelease['Tracks']:

                cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
        
                controlValueDict = {"TrackID":      track['id'],
                                    "ReleaseID":    rg['id']}

                newValueDict = {"ArtistID":         artistid,
                                "ArtistName":       artist['artist_name'],
                                "AlbumTitle":       rg['title'],
                                "AlbumASIN":        hybridrelease['AlbumASIN'],
                                "AlbumID":          rg['id'],
                                "TrackTitle":       track['title'],
                                "TrackDuration":    track['duration'],
                                "TrackNumber":      track['number'],
                                "CleanName":        cleanname
                            }
                            
                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
            
                if not match:
                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
                #if not match:
                    #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
                if match:
                    newValueDict['Location'] = match['Location']
                    newValueDict['BitRate'] = match['BitRate']
                    newValueDict['Format'] = match['Format']
                    #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location']))
                                
                myDB.upsert("alltracks", newValueDict, controlValueDict)
            
            # Delete matched tracks from the have table
            #myDB.action('DELETE from have WHERE Matched="True"')
            
            # If there's no release in the main albums tables, add the default (hybrid)
            # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
            # check if the album already exists
            rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
            if not rg_exists:
                releaseid = rg['id']
            else:
                releaseid = rg_exists['ReleaseID']
            
            album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone()

            controlValueDict = {"AlbumID":  rg['id']}

            newValueDict = {"ArtistID":         album['ArtistID'],
                            "ArtistName":       album['ArtistName'],
                            "AlbumTitle":       album['AlbumTitle'],
                            "ReleaseID":        album['ReleaseID'],
                            "AlbumASIN":        album['AlbumASIN'],
                            "ReleaseDate":      album['ReleaseDate'],
                            "Type":             album['Type'],
                            "ReleaseCountry":   album['ReleaseCountry'],
                            "ReleaseFormat":    album['ReleaseFormat']
                        }
                
            if not rg_exists:
                
                today = helpers.today()
                
                newValueDict['DateAdded']= today
                                
                if headphones.AUTOWANT_ALL:
                    newValueDict['Status'] = "Wanted"
                elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING:
                    newValueDict['Status'] = "Wanted"
                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
                # The first test just makes sure we have year-month-day
                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING:
                    newValueDict['Status'] = "Wanted"
                else:
                    newValueDict['Status'] = "Skipped"
            
            myDB.upsert("albums", newValueDict, controlValueDict) 

            tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()

            # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT
            total_track_count = len(tracks)
            
            for track in tracks:
            
                controlValueDict = {"TrackID":  track['TrackID'],
                                    "AlbumID":  rg['id']}

                newValueDict = {"ArtistID":     track['ArtistID'],
                            "ArtistName":       track['ArtistName'],
                            "AlbumTitle":       track['AlbumTitle'],
                            "AlbumASIN":        track['AlbumASIN'],
                            "ReleaseID":        track['ReleaseID'],
                            "TrackTitle":       track['TrackTitle'],
                            "TrackDuration":    track['TrackDuration'],
                            "TrackNumber":      track['TrackNumber'],
                            "CleanName":        track['CleanName'],
                            "Location":         track['Location'],
                            "Format":           track['Format'],
                            "BitRate":          track['BitRate']
                            }
                            
                myDB.upsert("tracks", newValueDict, controlValueDict) 

            # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
            have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']]))
            marked_as_downloaded = False
            
            if rg_exists:
                if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
                    marked_as_downloaded = True
            else:
                if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
                    marked_as_downloaded = True

            logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
            cache.getThumb(AlbumID=rg['id'])
            
            #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected:
            if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL:    
                from headphones import searcher
                searcher.searchforalbum(albumid=rg['id'])
        else:
            if skip_log == 0:
                logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title']))
Exemple #20
0
        controlValueDict = {"ReleaseID":  rg['id']}

        newValueDict = {"ArtistID":         artistid,
                        "ArtistName":       artist['artist_name'],
                        "AlbumTitle":       rg['title'],
                        "AlbumID":          rg['id'],
                        "AlbumASIN":        hybridrelease['AlbumASIN'],
                        "ReleaseDate":      hybridrelease['ReleaseDate'],
                        "Type":             rg['type']
                    }
                    
        myDB.upsert("allalbums", newValueDict, controlValueDict)
        
        for track in hybridrelease['Tracks']:

            cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
    
            controlValueDict = {"TrackID":      track['id'],
                                "ReleaseID":    rg['id']}

            newValueDict = {"ArtistID":         artistid,
                            "ArtistName":       artist['artist_name'],
                            "AlbumTitle":       rg['title'],
                            "AlbumASIN":        hybridrelease['AlbumASIN'],
                            "AlbumID":          rg['id'],
                            "TrackTitle":       track['title'],
                            "TrackDuration":    track['duration'],
                            "TrackNumber":      track['number'],
                            "CleanName":        cleanname
                        }
                        
Exemple #21
0
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False):


    if cron and not headphones.LIBRARYSCAN:
        return
        
    if not dir:
        if not headphones.MUSIC_DIR:
            return
        else:
            dir = headphones.MUSIC_DIR
    
    # If we're appending a dir, it's coming from the post processor which is
    # already bytestring
    if not append:
        dir = dir.encode(headphones.SYS_ENCODING)
        
    if not os.path.isdir(dir):
        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace'))
        return

    myDB = db.DBConnection()
    
    if not append:
        # Clean up bad filepaths
        tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL')

        for track in tracks:
            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING)
            if not os.path.isfile(encoded_track_string):
                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])
                myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])
    
        del_have_tracks = myDB.select('SELECT Location, Matched from have')

        for track in del_have_tracks:
            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING)
            if not os.path.isfile(encoded_track_string):
                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
                myDB.action('UPDATE have SET Matched=NULL WHERE Matched=?', [track['Matched']])
                logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace'))
           ###############myDB.action('DELETE from have')

    logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))

    new_artists = []
    bitrates = []
    
    song_list = []
    new_song_count = 0
    
    for r,d,f in os.walk(dir):
        #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one
        #using a list comprehension will not work correctly for nested subdirectories (os.walk keeps its original list)
        for directory in d[:]:
            if directory.startswith("."):
                d.remove(directory)
        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):

                song = os.path.join(r, files)

                # We need the unicode path to use for logging, inserting into database
                unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace')

                # Try to read the metadata
                try:
                    f = MediaFile(song)

                except:
                    logger.error('Cannot read file: ' + unicode_song_path)
                    continue
                    
                # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)
                    
                # Use the album artist over the artist if available
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    f_artist = None
                    
                # Add the song to our song list - 
                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)

                if f_artist and f.album and f.title:
                    CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title)
                else:
                    CleanName = None

                controlValueDict = {'Location'   : unicode_song_path}

                newValueDict = { 'TrackID' : f.mb_trackid,
                              #'ReleaseID' : f.mb_albumid,
                              'ArtistName' : f_artist,
                              'AlbumTitle' : f.album,
                              'TrackNumber': f.track,
                              'TrackLength': f.length,
                              'Genre'      : f.genre,
                              'Date'       : f.date,
                              'TrackTitle' : f.title,
                              'BitRate'    : f.bitrate,
                              'Format'     : f.format,
                              'CleanName'  : CleanName
                              }
                              
                #song_list.append(song_dict)
                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone()
                #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
                if not check_exist_song:
                    myDB.upsert("have", newValueDict, controlValueDict)
                    new_song_count+=1
                elif check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
                    newValueDict['Matched'] = None
                    myDB.upsert("have", newValueDict, controlValueDict)
                    new_song_count+=1


    # Now we start track matching
    logger.info("%s new/modified songs found and added to the database" % new_song_count)
    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"])
    total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0]
    logger.info("Found " + str(total_number_of_songs) + " unmatched tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
    
    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
    
    ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])
    
    # We'll use this to give a % completion, just because the track matching might take a while
    song_count = 0
    latest_artist = []
    
    for song in song_list:

        latest_artist.append(song['ArtistName'])
        if song_count == 0:
            logger.info("Now matching songs by %s" % song['ArtistName'])
        elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0:
            logger.info("Now matching songs by %s" % song['ArtistName'])
        
        #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle']
        song_count += 1
        completion_percentage = float(song_count)/total_number_of_songs * 100
        
        if completion_percentage%10 == 0:
            logger.info("Track matching is " + str(completion_percentage) + "% complete")
        
        #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED.  WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT
        #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME.  ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION,
        #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
        #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.

        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
            
            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
            if track:
                controlValueDict = { 'ArtistName' : track['ArtistName'],
                                     'AlbumTitle' : track['AlbumTitle'],
                                     'TrackTitle' : track['TrackTitle'] }                           
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }
                myDB.upsert("tracks", newValueDict, controlValueDict)

                controlValueDict2 = { 'ArtistName' : song['ArtistName'],
                                     'AlbumTitle' : song['AlbumTitle'],
                                     'TrackTitle' : song['TrackTitle'] }
                newValueDict2 = { 'Matched' : track['AlbumID']}
                myDB.upsert("have", newValueDict2, controlValueDict2)
            else:
                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
                if track:
                    controlValueDict = { 'CleanName' : track['CleanName']}                         
                    newValueDict = { 'Location' : song['Location'],
                                     'BitRate'  : song['BitRate'],
                                     'Format'   : song['Format'] }               
                    myDB.upsert("tracks", newValueDict, controlValueDict)

                    controlValueDict2 = { 'CleanName' : song['CleanName']}
                    newValueDict2 = { 'Matched' : track['AlbumID']}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
  

            alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()  
            if alltrack:
                controlValueDict = { 'ArtistName' : alltrack['ArtistName'],
                                     'AlbumTitle' : alltrack['AlbumTitle'],
                                     'TrackTitle' : alltrack['TrackTitle'] }
                newValueDict = { 'Location' : song['Location'],
                                 'BitRate'  : song['BitRate'],
                                 'Format'   : song['Format'] }
                myDB.upsert("alltracks", newValueDict, controlValueDict)

                controlValueDict2 = { 'ArtistName' : song['ArtistName'],
                                     'AlbumTitle' : song['AlbumTitle'],
                                     'TrackTitle' : song['TrackTitle'] } 
                newValueDict2 = { 'Matched' : alltrack['AlbumID']}
                myDB.upsert("have", newValueDict2, controlValueDict2)
            else:
                alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
                if alltrack:
                    controlValueDict = { 'CleanName' : alltrack['CleanName']}                                         
                    newValueDict = { 'Location' : song['Location'],
                                     'BitRate'  : song['BitRate'],
                                     'Format'   : song['Format'] }               
                    myDB.upsert("alltracks", newValueDict, controlValueDict)

                    controlValueDict2 = { 'CleanName' : song['CleanName']}
                    newValueDict2 = { 'Matched' : alltrack['AlbumID']}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
   
        
        # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
        if song['ArtistName']:
            new_artists.append(song['ArtistName'])
        else:
            continue
        
        #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])

    logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))

    
    if not append:
        # Clean up the new artist list
        unique_artists = {}.fromkeys(new_artists).keys()
        current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')

        #There was a bug where artists with special characters (-,') would show up in new artists.     
        artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]]
        
        # Update track counts
        logger.info('Updating current artist track counts')
    
        for artist in current_artists:
            # Have tracks are selected from tracks table and not all tracks because of duplicates
            # We update the track count upon an album switch to compliment this
            havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['ArtistName']]))
            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']])
            
        logger.info('Found %i new artists' % len(artist_list))
    
        if len(artist_list):
            if headphones.ADD_ARTISTS:
                logger.info('Importing %i new artists' % len(artist_list))
                importer.artistlist_to_mbids(artist_list)
            else:
                logger.info('To add these artists, go to Manage->Manage New Artists')
                myDB.action('DELETE from newartists')
                for artist in artist_list:
                    myDB.action('INSERT into newartists VALUES (?)', [artist])
        
        if headphones.DETECT_BITRATE:
            headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
            
    else:
        # If we're appending a new album to the database, update the artists total track counts
        logger.info('Updating artist track counts')
        
        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [ArtistName]))
        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])

    update_album_status()
    logger.info('Library scan complete')
Exemple #22
0
            "AlbumASIN": release_dict["asin"],
            "ReleaseDate": release_dict["date"],
            "DateAdded": helpers.today(),
            "Status": "Wanted",
            "Type": release_dict["rg_type"],
        }

        myDB.upsert("albums", newValueDict, controlValueDict)

        # keep a local cache of these so that external programs that are adding releasesByID don't hammer MB
        myDB.action("INSERT INTO releases VALUES( ?, ?)", [rid, release_dict["rgid"]])

        for track in release_dict["tracks"]:

            cleanname = helpers.cleanName(
                release_dict["artist_name"] + " " + release_dict["rg_title"] + " " + track["title"]
            )

            controlValueDict = {"TrackID": track["id"], "AlbumID": rgid}
            newValueDict = {
                "ArtistID": release_dict["artist_id"],
                "ArtistName": release_dict["artist_name"],
                "AlbumTitle": release_dict["rg_title"],
                "AlbumASIN": release_dict["asin"],
                "TrackTitle": track["title"],
                "TrackDuration": track["duration"],
                "TrackNumber": track["number"],
                "CleanName": cleanname,
            }

            match = myDB.action("SELECT Location, BitRate, Format from have WHERE CleanName=?", [cleanname]).fetchone()
Exemple #23
0
def libraryScan(dir=None):

    if not dir:
        dir = headphones.MUSIC_DIR

    try:
        dir = str(dir)
    except UnicodeEncodeError:
        dir = unicode(dir).encode("unicode_escape")

    if not os.path.isdir(dir):
        logger.warn("Cannot find directory: %s. Not scanning" % dir)
        return

    myDB = db.DBConnection()

    # Clean up bad filepaths
    tracks = myDB.select("SELECT Location, TrackID from tracks WHERE Location IS NOT NULL")

    for track in tracks:
        if not os.path.isfile(track["Location"].encode(headphones.SYS_ENCODING)):
            myDB.action(
                "UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?",
                [None, None, None, track["TrackID"]],
            )

    logger.info("Scanning music directory: %s" % dir)

    new_artists = []
    bitrates = []

    myDB.action("DELETE from have")

    for r, d, f in os.walk(dir):
        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith("." + x.lower()) for x in headphones.MEDIA_FORMATS):

                song = os.path.join(r, files)
                file = unicode(os.path.join(r, files), headphones.SYS_ENCODING, errors="replace")

                # Try to read the metadata
                try:
                    f = MediaFile(song)

                except:
                    logger.error("Cannot read file: " + file)
                    continue

                    # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)

                    # Try to find a match based on artist/album/tracktitle
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    continue

                if f_artist and f.album and f.title:

                    track = myDB.action(
                        "SELECT TrackID from tracks WHERE CleanName LIKE ?",
                        [helpers.cleanName(f_artist + " " + f.album + " " + f.title)],
                    ).fetchone()

                    if not track:
                        track = myDB.action(
                            "SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?",
                            [f_artist, f.album, f.title],
                        ).fetchone()

                    if track:
                        myDB.action(
                            "UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?",
                            [file, f.bitrate, f.format, track["TrackID"]],
                        )
                        continue

                        # Try to match on mbid if available and we couldn't find a match based on metadata
                if f.mb_trackid:

                    # Wondering if theres a better way to do this -> do one thing if the row exists,
                    # do something else if it doesn't
                    track = myDB.action("SELECT TrackID from tracks WHERE TrackID=?", [f.mb_trackid]).fetchone()

                    if track:
                        myDB.action(
                            "UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?",
                            [file, f.bitrate, f.format, track["TrackID"]],
                        )
                        continue

                        # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
                new_artists.append(f_artist)

                # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database
                myDB.action(
                    "INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                    [
                        f_artist,
                        f.album,
                        f.track,
                        f.title,
                        f.length,
                        f.bitrate,
                        f.genre,
                        f.date,
                        f.mb_trackid,
                        file,
                        helpers.cleanName(f_artist + " " + f.album + " " + f.title),
                        f.format,
                    ],
                )

    logger.info("Completed scanning of directory: %s" % dir)
    logger.info("Checking filepaths to see if we can find any matches")

    # Now check empty file paths to see if we can find a match based on their folder format
    tracks = myDB.select("SELECT * from tracks WHERE Location IS NULL")
    for track in tracks:

        release = myDB.action("SELECT * from albums WHERE AlbumID=?", [track["AlbumID"]]).fetchone()

        try:
            year = release["ReleaseDate"][:4]
        except TypeError:
            year = ""

        artist = release["ArtistName"].replace("/", "_")
        album = release["AlbumTitle"].replace("/", "_")
        releasetype = release["Type"].replace("/", "_")

        if release["ArtistName"].startswith("The "):
            sortname = release["ArtistName"][4:]
        else:
            sortname = release["ArtistName"]

        if sortname.isdigit():
            firstchar = "0-9"
        else:
            firstchar = sortname[0]

        albumvalues = {
            "$Artist": artist,
            "$Album": album,
            "$Year": year,
            "$Type": releasetype,
            "$First": firstchar,
            "$artist": artist.lower(),
            "$album": album.lower(),
            "$year": year,
            "$type": releasetype.lower(),
            "$first": firstchar.lower(),
        }

        folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues)
        folder = folder.replace("./", "_/").replace(":", "_").replace("?", "_")

        if folder.endswith("."):
            folder = folder.replace(folder[len(folder) - 1], "_")

        if not track["TrackNumber"]:
            tracknumber = ""
        else:
            tracknumber = "%02d" % track["TrackNumber"]

        title = track["TrackTitle"]

        trackvalues = {
            "$Track": tracknumber,
            "$Title": title,
            "$Artist": release["ArtistName"],
            "$Album": release["AlbumTitle"],
            "$Year": year,
            "$track": tracknumber,
            "$title": title.lower(),
            "$artist": release["ArtistName"].lower(),
            "$album": release["AlbumTitle"].lower(),
            "$year": year,
        }

        new_file_name = helpers.replace_all(headphones.FILE_FORMAT, trackvalues).replace("/", "_") + ".*"

        new_file_name = new_file_name.replace("?", "_").replace(":", "_")

        full_path_to_file = os.path.normpath(os.path.join(headphones.MUSIC_DIR, folder, new_file_name)).encode(
            headphones.SYS_ENCODING, "replace"
        )

        match = glob.glob(full_path_to_file)

        if match:

            logger.info("Found a match: %s. Writing MBID to metadata" % match[0])

            unipath = unicode(match[0], headphones.SYS_ENCODING, errors="replace")

            myDB.action("UPDATE tracks SET Location=? WHERE TrackID=?", [unipath, track["TrackID"]])
            myDB.action("DELETE from have WHERE Location=?", [unipath])

            # Try to insert the appropriate track id so we don't have to keep doing this
            try:
                f = MediaFile(match[0])
                f.mb_trackid = track["TrackID"]
                f.save()
                myDB.action(
                    "UPDATE tracks SET BitRate=?, Format=? WHERE TrackID=?", [f.bitrate, f.format, track["TrackID"]]
                )

                logger.debug("Wrote mbid to track: %s" % match[0])

            except:
                logger.error("Error embedding track id into: %s" % match[0])
                continue

    logger.info("Done checking empty filepaths")
    logger.info("Done syncing library with directory: %s" % dir)

    # Clean up the new artist list
    unique_artists = {}.fromkeys(new_artists).keys()
    current_artists = myDB.select("SELECT ArtistName, ArtistID from artists")

    artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]]

    # Update track counts
    logger.info("Updating track counts")

    for artist in current_artists:
        havetracks = len(
            myDB.select(
                "SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL", [artist["ArtistID"]]
            )
        ) + len(myDB.select("SELECT TrackTitle from have WHERE ArtistName like ?", [artist["ArtistName"]]))
        myDB.action("UPDATE artists SET HaveTracks=? WHERE ArtistID=?", [havetracks, artist["ArtistID"]])

    logger.info("Found %i new artists" % len(artist_list))

    if len(artist_list):
        if headphones.ADD_ARTISTS:
            logger.info("Importing %i new artists" % len(artist_list))
            importer.artistlist_to_mbids(artist_list)
        else:
            logger.info("To add these artists, go to Manage->Manage New Artists")
            headphones.NEW_ARTISTS = artist_list

    if headphones.DETECT_BITRATE:
        headphones.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000
Exemple #24
0
def get_new_releases(rgid,includeExtras=False,forcefull=False):

    myDB = db.DBConnection()
    results = []
    try:
        limit = 100
        newResults = None
        while newResults == None or len(newResults) >= limit:
            newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results))
            if 'release-list' not in newResults:
                break #may want to raise an exception here instead ?
            newResults = newResults['release-list']
            results += newResults

    except musicbrainzngs.WebServiceError as e:
        logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e)))
        time.sleep(5)
        return False

    if not results or len(results) == 0:
        return False

    #Clean all references to releases in dB that are no longer referenced in musicbrainz
    release_list = []
    force_repackage1 = 0
    if len(results) != 0:
        for release_mark in results:
            release_list.append(unicode(release_mark['id']))
            release_title = release_mark['title']
        remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid])
        if remove_missing_releases:
            for items in remove_missing_releases:
                if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid:
                    # Remove all from albums/tracks that aren't in release
                    myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']])
                    myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']])
                    myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']])
                    myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']])
                    logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID'])
                    force_repackage1 = 1
    else:
        logger.info("There was either an error pulling data from MusicBrainz or there might not be any releases for this category")

    num_new_releases = 0

    for releasedata in results:
        #releasedata.get will return None if it doesn't have a status
        #all official releases should have the Official status included
        if not includeExtras and releasedata.get('status') != 'Official':
            continue

        release = {}
        rel_id_check = releasedata['id']
        artistid = unicode(releasedata['artist-credit'][0]['artist']['id'])

        album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone()
        if not album_checker or forcefull:
            #DELETE all references to this release since we're updating it anyway.
            myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check])
            myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check])
            release['AlbumTitle'] = unicode(releasedata['title'])
            release['AlbumID'] = unicode(rgid)
            release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None
            release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None
            release['ReleaseID'] = releasedata['id']
            if 'release-group' not in releasedata:
                raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid)
            release['Type'] = unicode(releasedata['release-group']['type'])

            if release['Type'] == 'Album' and 'secondary-type-list' in releasedata['release-group']:
                secondary_type = unicode(releasedata['release-group']['secondary-type-list'][0])
                if secondary_type != release['Type']:
                    release['Type'] = secondary_type

            #making the assumption that the most important artist will be first in the list
            if 'artist-credit' in releasedata:
                release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id'])
                release['ArtistName'] = unicode(releasedata['artist-credit-phrase'])
            else:
                logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.')
                return False


            release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown'
            #assuming that the list will contain media and that the format will be consistent
            try:
                additional_medium=''
                for position in releasedata['medium-list']:
                    if position['format'] == releasedata['medium-list'][0]['format']:
                        medium_count = int(position['position'])
                    else:
                        additional_medium = additional_medium+' + '+position['format']
                if medium_count == 1:
                    disc_number = ''
                else:
                    disc_number = str(medium_count)+'x'
                packaged_medium = disc_number+releasedata['medium-list'][0]['format']+additional_medium
                release['ReleaseFormat'] = unicode(packaged_medium)
            except:
                release['ReleaseFormat'] = u'Unknown'

            release['Tracks'] = getTracksFromRelease(releasedata)

        # What we're doing here now is first updating the allalbums & alltracks table to the most
        # current info, then moving the appropriate release into the album table and its associated
        # tracks into the tracks table
            controlValueDict = {"ReleaseID" : release['ReleaseID']}

            newValueDict = {"ArtistID":         release['ArtistID'],
                            "ArtistName":       release['ArtistName'],
                            "AlbumTitle":       release['AlbumTitle'],
                            "AlbumID":          release['AlbumID'],
                            "AlbumASIN":        release['AlbumASIN'],
                            "ReleaseDate":      release['ReleaseDate'],
                            "Type":             release['Type'],
                            "ReleaseCountry":   release['ReleaseCountry'],
                            "ReleaseFormat":    release['ReleaseFormat']
                        }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            for track in release['Tracks']:

                cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title'])

                controlValueDict = {"TrackID":      track['id'],
                                    "ReleaseID":    release['ReleaseID']}

                newValueDict = {"ArtistID":         release['ArtistID'],
                                "ArtistName":       release['ArtistName'],
                                "AlbumTitle":       release['AlbumTitle'],
                                "AlbumID":          release['AlbumID'],
                                "AlbumASIN":        release['AlbumASIN'],
                                "TrackTitle":       track['title'],
                                "TrackDuration":    track['duration'],
                                "TrackNumber":      track['number'],
                                "CleanName":        cleanname
                            }

                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()

                if not match:
                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone()
                #if not match:
                    #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                if match:
                    newValueDict['Location'] = match['Location']
                    newValueDict['BitRate'] = match['BitRate']
                    newValueDict['Format'] = match['Format']
                    #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location']))

                myDB.upsert("alltracks", newValueDict, controlValueDict)
            num_new_releases = num_new_releases + 1
            #print releasedata['title']
            #print num_new_releases
            if album_checker:
                logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
            else:
                logger.info('[%s] New release %s (%s) added' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
    if force_repackage1 == 1:
        num_new_releases = -1
        logger.info('[%s] Forcing repackage of %s, since dB releases have been removed' % (release['ArtistName'], release_title))
    else:
        num_new_releases = num_new_releases

    return num_new_releases
Exemple #25
0
def libraryScan(dir=None):

    if not dir:
        dir = headphones.MUSIC_DIR

    try:
        dir = str(dir)
    except UnicodeEncodeError:
        dir = unicode(dir).encode('unicode_escape')

    if not os.path.isdir(dir):
        logger.warn('Cannot find directory: %s. Not scanning' % dir)
        return

    myDB = db.DBConnection()

    # Clean up bad filepaths
    tracks = myDB.select(
        'SELECT Location, TrackID from tracks WHERE Location IS NOT NULL')

    for track in tracks:
        if not os.path.isfile(track['Location'].encode(
                headphones.SYS_ENCODING)):
            myDB.action(
                'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?',
                [None, None, None, track['TrackID']])

    logger.info('Scanning music directory: %s' % dir)

    new_artists = []
    bitrates = []

    myDB.action('DELETE from have')

    for r, d, f in os.walk(dir):
        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith('.' + x.lower())
                   for x in headphones.MEDIA_FORMATS):

                song = os.path.join(r, files)
                file = unicode(os.path.join(r, files),
                               headphones.SYS_ENCODING,
                               errors='replace')

                # Try to read the metadata
                try:
                    f = MediaFile(song)

                except:
                    logger.error('Cannot read file: ' + file)
                    continue

                # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)

                # Try to find a match based on artist/album/tracktitle
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    continue

                if f_artist and f.album and f.title:

                    track = myDB.action(
                        'SELECT TrackID from tracks WHERE CleanName LIKE ?', [
                            helpers.cleanName(f_artist + ' ' + f.album + ' ' +
                                              f.title)
                        ]).fetchone()

                    if not track:
                        track = myDB.action(
                            'SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                            [f_artist, f.album, f.title]).fetchone()

                    if track:
                        myDB.action(
                            'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?',
                            [file, f.bitrate, f.format, track['TrackID']])
                        continue

                # Try to match on mbid if available and we couldn't find a match based on metadata
                if f.mb_trackid:

                    # Wondering if theres a better way to do this -> do one thing if the row exists,
                    # do something else if it doesn't
                    track = myDB.action(
                        'SELECT TrackID from tracks WHERE TrackID=?',
                        [f.mb_trackid]).fetchone()

                    if track:
                        myDB.action(
                            'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?',
                            [file, f.bitrate, f.format, track['TrackID']])
                        continue

                # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
                new_artists.append(f_artist)

                # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database
                myDB.action(
                    'INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
                    [
                        f_artist, f.album, f.track, f.title, f.length,
                        f.bitrate, f.genre, f.date, f.mb_trackid, file,
                        helpers.cleanName(f_artist + ' ' + f.album + ' ' +
                                          f.title), f.format
                    ])

    logger.info('Completed scanning of directory: %s' % dir)
    logger.info('Checking filepaths to see if we can find any matches')

    # Now check empty file paths to see if we can find a match based on their folder format
    tracks = myDB.select('SELECT * from tracks WHERE Location IS NULL')
    for track in tracks:

        release = myDB.action('SELECT * from albums WHERE AlbumID=?',
                              [track['AlbumID']]).fetchone()

        try:
            year = release['ReleaseDate'][:4]
        except TypeError:
            year = ''

        artist = release['ArtistName'].replace('/', '_')
        album = release['AlbumTitle'].replace('/', '_')
        releasetype = release['Type'].replace('/', '_')

        if release['ArtistName'].startswith('The '):
            sortname = release['ArtistName'][4:]
        else:
            sortname = release['ArtistName']

        if sortname.isdigit():
            firstchar = '0-9'
        else:
            firstchar = sortname[0]

        albumvalues = {
            '$Artist': artist,
            '$Album': album,
            '$Year': year,
            '$Type': releasetype,
            '$First': firstchar,
            '$artist': artist.lower(),
            '$album': album.lower(),
            '$year': year,
            '$type': releasetype.lower(),
            '$first': firstchar.lower()
        }

        folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues)
        folder = folder.replace('./', '_/').replace(':', '_').replace('?', '_')

        if folder.endswith('.'):
            folder = folder.replace(folder[len(folder) - 1], '_')

        if not track['TrackNumber']:
            tracknumber = ''
        else:
            tracknumber = '%02d' % track['TrackNumber']

        title = track['TrackTitle']

        trackvalues = {
            '$Track': tracknumber,
            '$Title': title,
            '$Artist': release['ArtistName'],
            '$Album': release['AlbumTitle'],
            '$Year': year,
            '$track': tracknumber,
            '$title': title.lower(),
            '$artist': release['ArtistName'].lower(),
            '$album': release['AlbumTitle'].lower(),
            '$year': year
        }

        new_file_name = helpers.replace_all(
            headphones.FILE_FORMAT, trackvalues).replace('/', '_') + '.*'

        new_file_name = new_file_name.replace('?', '_').replace(':', '_')

        full_path_to_file = os.path.normpath(
            os.path.join(headphones.MUSIC_DIR, folder,
                         new_file_name)).encode(headphones.SYS_ENCODING,
                                                'replace')

        match = glob.glob(full_path_to_file)

        if match:

            logger.info('Found a match: %s. Writing MBID to metadata' %
                        match[0])

            unipath = unicode(match[0],
                              headphones.SYS_ENCODING,
                              errors='replace')

            myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?',
                        [unipath, track['TrackID']])
            myDB.action('DELETE from have WHERE Location=?', [unipath])

            # Try to insert the appropriate track id so we don't have to keep doing this
            try:
                f = MediaFile(match[0])
                f.mb_trackid = track['TrackID']
                f.save()
                myDB.action(
                    'UPDATE tracks SET BitRate=?, Format=? WHERE TrackID=?',
                    [f.bitrate, f.format, track['TrackID']])

                logger.debug('Wrote mbid to track: %s' % match[0])

            except:
                logger.error('Error embedding track id into: %s' % match[0])
                continue

    logger.info('Done checking empty filepaths')
    logger.info('Done syncing library with directory: %s' % dir)

    # Clean up the new artist list
    unique_artists = {}.fromkeys(new_artists).keys()
    current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')

    artist_list = [
        f for f in unique_artists
        if f.lower() not in [x[0].lower() for x in current_artists]
    ]

    # Update track counts
    logger.info('Updating track counts')

    for artist in current_artists:
        havetracks = len(
            myDB.select(
                'SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL',
                [artist['ArtistID']])) + len(
                    myDB.select(
                        'SELECT TrackTitle from have WHERE ArtistName like ?',
                        [artist['ArtistName']]))
        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?',
                    [havetracks, artist['ArtistID']])

    logger.info('Found %i new artists' % len(artist_list))

    if len(artist_list):
        if headphones.ADD_ARTISTS:
            logger.info('Importing %i new artists' % len(artist_list))
            importer.artistlist_to_mbids(artist_list)
        else:
            logger.info(
                'To add these artists, go to Manage->Manage New Artists')
            headphones.NEW_ARTISTS = artist_list

    if headphones.DETECT_BITRATE:
        headphones.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000