Пример #1
0
 def test_clean_name(self):
     """helpers: check correctness of clean_name() function"""
     cases = {
         u' Weiße & rose ':
         'Weisse and rose',
         u'Multiple / spaces':
         'Multiple spaces',
         u'Kevin\'s m²':
         'Kevins m2',
         u'Symphonęy Nº9':
         'Symphoney No.9',
         u'ÆæßðÞIJij':
         u'AeaessdThIJıj',
         u'Obsessió (Cerebral Apoplexy remix)':
         'obsessio cerebral '
         'apoplexy remix',
         u'Doktór Hałabała i siedmiu zbojów':
         'doktor halabala i siedmiu '
         'zbojow',
         u'Arbetets Söner och Döttrar':
         'arbetets soner och dottrar',
         u'Björk Guðmundsdóttir':
         'bjork gudmundsdottir',
         u'L\'Arc~en~Ciel':
         'larc en ciel',
         u'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)':
         u'Orquesta de la Luz オルケスタ デ ラ ルス'
     }
     for first, second in cases.iteritems():
         nf = clean_name(first).lower()
         ns = clean_name(second).lower()
         self.assertEqual(nf, ns, u"check cleaning of case (%s,"
                          u"%s)" % (nf, ns))
Пример #2
0
    def test_clean_name(self):
        """helpers: check correctness of clean_name() function"""
        cases = {
            u' Weiße & rose ': 'Weisse and rose',
            u'Multiple / spaces': 'Multiple spaces',
            u'Kevin\'s m²': 'Kevins m2',
            u'Symphonęy Nº9': 'Symphoney No.9',
            u'ÆæßðÞIJij': u'AeaessdThIJıj',
            u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral '
                                                    'apoplexy remix',
            u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu '
                                                    'zbojow',
            u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar',
            u'Björk Guðmundsdóttir': 'bjork gudmundsdottir',
            u'L\'Arc~en~Ciel': 'larc en ciel',
            u'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)':
                u'Orquesta de la Luz オルケスタ デ ラ ルス'

        }
        for first, second in cases.iteritems():
            nf = clean_name(first).lower()
            ns = clean_name(second).lower()
            self.assertEqual(
                nf, ns, u"check cleaning of case (%s,"
                        u"%s)" % (nf, ns)
            )
Пример #3
0
 def test_clean_name_nonunicode(self):
     """helpers: check if clean_name() works on non-unicode input"""
     input = 'foo $ bar/BAZ'
     test = clean_name(input).lower()
     expected = 'foo bar baz'
     self.assertEqual(test, expected,
                      "check clean_name() works on non-unicode")
     input = 'fóó $ BAZ'
     test = clean_name(input).lower()
     expected = clean_name('%fóó baz ').lower()
     self.assertEqual(test, expected,
                      "check clean_name() with narrow non-ascii input")
Пример #4
0
 def test_clean_name_nonunicode(self):
     """helpers: check if clean_name() works on non-unicode input"""
     input = 'foo $ bar/BAZ'
     test = clean_name(input).lower()
     expected = 'foo bar baz'
     self.assertEqual(
         test, expected, "check clean_name() works on non-unicode"
     )
     input = 'fóó $ BAZ'
     test = clean_name(input).lower()
     expected = clean_name('%fóó baz ').lower()
     self.assertEqual(
         test, expected, "check clean_name() with narrow non-ascii input"
     )
Пример #5
0
def libraryScan(dir=None,
                append=False,
                ArtistID=None,
                ArtistName=None,
                cron=False,
                artistScan=False):
    if cron and not headphones.CONFIG.LIBRARYSCAN:
        return

    if not dir:
        if not headphones.CONFIG.MUSIC_DIR:
            return
        else:
            dir = headphones.CONFIG.MUSIC_DIR

    # If we're appending a dir, it's coming from the post processor which is
    # already bytestring
    if not append or artistScan:
        dir = dir.encode(headphones.SYS_ENCODING)

    if not os.path.isdir(dir):
        logger.warn('Cannot find directory: %s. Not scanning' %
                    dir.decode(headphones.SYS_ENCODING, 'replace'))
        return

    myDB = db.DBConnection()
    new_artists = []

    logger.info('Scanning music directory: %s' %
                dir.decode(headphones.SYS_ENCODING, 'replace'))

    if not append:

        # Clean up bad filepaths. Queries can take some time, ensure all results are loaded before processing
        if ArtistID:
            tracks = myDB.action(
                'SELECT Location FROM alltracks WHERE ArtistID = ? AND Location IS NOT NULL UNION SELECT Location FROM tracks WHERE ArtistID = ? AND Location '
                'IS NOT NULL', [ArtistID, ArtistID])
        else:
            tracks = myDB.action(
                'SELECT Location FROM alltracks WHERE Location IS NOT NULL UNION SELECT Location FROM tracks WHERE Location IS NOT NULL'
            )

        locations = []
        for track in tracks:
            locations.append(track['Location'])
        for location in locations:
            encoded_track_string = location.encode(headphones.SYS_ENCODING,
                                                   'replace')
            if not os.path.isfile(encoded_track_string):
                myDB.action(
                    'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                    [None, None, None, location])
                myDB.action(
                    'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                    [None, None, None, location])

        if ArtistName:
            del_have_tracks = myDB.select(
                'SELECT Location, Matched, ArtistName FROM have WHERE ArtistName = ? COLLATE NOCASE',
                [ArtistName])
        else:
            del_have_tracks = myDB.select(
                'SELECT Location, Matched, ArtistName FROM have')

        locations = []
        for track in del_have_tracks:
            locations.append([track['Location'], track['ArtistName']])
        for location in locations:
            encoded_track_string = location[0].encode(headphones.SYS_ENCODING,
                                                      'replace')
            if not os.path.isfile(encoded_track_string):
                if location[1]:
                    # Make sure deleted files get accounted for when updating artist track counts
                    new_artists.append(location[1])
                myDB.action('DELETE FROM have WHERE Location=?', [location[0]])
                logger.info(
                    'File %s removed from Headphones, as it is no longer on disk'
                    % encoded_track_string.decode(headphones.SYS_ENCODING,
                                                  'replace'))

    bitrates = []
    song_list = []
    latest_subdirectory = []

    new_song_count = 0
    file_count = 0

    for r, d, f in helpers.walk_directory(dir):
        # Filter paths based on config. Note that these methods work directly
        # on the inputs
        helpers.path_filter_patterns(d, headphones.CONFIG.IGNORED_FOLDERS, r)
        helpers.path_filter_patterns(f, headphones.CONFIG.IGNORED_FILES, r)

        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith('.' + x.lower())
                   for x in headphones.MEDIA_FORMATS):
                subdirectory = r.replace(dir, '')
                latest_subdirectory.append(subdirectory)

                if file_count == 0 and r.replace(dir, '') != '':
                    logger.info(
                        "[%s] Now scanning subdirectory %s" %
                        (dir.decode(headphones.SYS_ENCODING, 'replace'),
                         subdirectory.decode(headphones.SYS_ENCODING,
                                             'replace')))
                elif latest_subdirectory[file_count] != latest_subdirectory[
                        file_count - 1] and file_count != 0:
                    logger.info(
                        "[%s] Now scanning subdirectory %s" %
                        (dir.decode(headphones.SYS_ENCODING, 'replace'),
                         subdirectory.decode(headphones.SYS_ENCODING,
                                             'replace')))

                song = os.path.join(r, files)

                # We need the unicode path to use for logging, inserting into database
                unicode_song_path = song.decode(headphones.SYS_ENCODING,
                                                'replace')

                # Try to read the metadata
                try:
                    f = MediaFile(song)
                except (FileTypeError, UnreadableFileError):
                    logger.warning(
                        "Cannot read media file '%s', skipping. It may be corrupted or not a media file.",
                        unicode_song_path)
                    continue
                except IOError:
                    logger.warning(
                        "Cannnot read media file '%s', skipping. Does the file exists?",
                        unicode_song_path)
                    continue

                # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)

                # Use the album artist over the artist if available
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    f_artist = None

                # Add the song to our song list -
                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)

                if f_artist and f.album and f.title:
                    CleanName = helpers.clean_name(f_artist + ' ' + f.album +
                                                   ' ' + f.title)
                else:
                    CleanName = None

                controlValueDict = {'Location': unicode_song_path}

                newValueDict = {
                    'TrackID': f.mb_trackid,
                    # 'ReleaseID' : f.mb_albumid,
                    'ArtistName': f_artist,
                    'AlbumTitle': f.album,
                    'TrackNumber': f.track,
                    'TrackLength': f.length,
                    'Genre': f.genre,
                    'Date': f.date,
                    'TrackTitle': f.title,
                    'BitRate': f.bitrate,
                    'Format': f.format,
                    'CleanName': CleanName
                }

                # song_list.append(song_dict)
                check_exist_song = myDB.action(
                    "SELECT * FROM have WHERE Location=?",
                    [unicode_song_path]).fetchone()
                # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
                if not check_exist_song:
                    # This is a new track
                    if f_artist:
                        new_artists.append(f_artist)
                    myDB.upsert("have", newValueDict, controlValueDict)
                    new_song_count += 1
                else:
                    if check_exist_song[
                            'ArtistName'] != f_artist or check_exist_song[
                                'AlbumTitle'] != f.album or check_exist_song[
                                    'TrackTitle'] != f.title:
                        # Important track metadata has been modified, need to run matcher again
                        if f_artist and f_artist != check_exist_song[
                                'ArtistName']:
                            new_artists.append(f_artist)
                        elif f_artist and f_artist == check_exist_song['ArtistName'] and \
                                        check_exist_song['Matched'] != "Ignored":
                            new_artists.append(f_artist)
                        else:
                            continue

                        newValueDict['Matched'] = None
                        myDB.upsert("have", newValueDict, controlValueDict)
                        myDB.action(
                            'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                            [None, None, None, unicode_song_path])
                        myDB.action(
                            'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                            [None, None, None, unicode_song_path])
                        new_song_count += 1
                    else:
                        # This track information hasn't changed
                        if f_artist and check_exist_song[
                                'Matched'] != "Ignored":
                            new_artists.append(f_artist)

                file_count += 1

    # Now we start track matching
    logger.info("%s new/modified songs found and added to the database" %
                new_song_count)
    song_list = myDB.action(
        "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
        [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"])
    total_number_of_songs = \
        myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
                    [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0]
    logger.info("Found " + str(total_number_of_songs) +
                " new/modified tracks in: '" +
                dir.decode(headphones.SYS_ENCODING, 'replace') +
                "'. Matching tracks to the appropriate releases....")

    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches

    # song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])

    # We'll use this to give a % completion, just because the track matching might take a while
    song_count = 0
    latest_artist = []
    last_completion_percentage = 0
    prev_artist_name = None
    artistid = None

    for song in song_list:

        latest_artist.append(song['ArtistName'])
        if song_count == 0:
            logger.info("Now matching songs by %s" % song['ArtistName'])
        elif latest_artist[song_count] != latest_artist[song_count -
                                                        1] and song_count != 0:
            logger.info("Now matching songs by %s" % song['ArtistName'])

        song_count += 1
        completion_percentage = math.floor(
            float(song_count) / total_number_of_songs * 1000) / 10

        if completion_percentage >= (last_completion_percentage + 10):
            logger.info("Track matching is " + str(completion_percentage) +
                        "% complete")
            last_completion_percentage = completion_percentage

        # THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED.  WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT
        # EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME.  ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION,
        # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
        # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.

        albumid = None

        if song['ArtistName'] and song['CleanName']:
            artist_name = song['ArtistName']
            clean_name = song['CleanName']

            # Only update if artist is in the db
            if artist_name != prev_artist_name:
                prev_artist_name = artist_name
                artistid = None

                artist_lookup = "\"" + artist_name.replace("\"", "\"\"") + "\""

                try:
                    dbartist = myDB.select(
                        'SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE '
                        + artist_lookup + '')
                except:
                    dbartist = None
                if not dbartist:
                    dbartist = myDB.select(
                        'SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName = ?',
                        [clean_name])
                    if not dbartist:
                        dbartist = myDB.select(
                            'SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName = ?',
                            [clean_name])
                        if not dbartist:
                            clean_artist = helpers.clean_name(artist_name)
                            if clean_artist:
                                dbartist = myDB.select(
                                    'SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
                                    [clean_artist, clean_artist + '{'])
                                if not dbartist:
                                    dbartist = myDB.select(
                                        'SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
                                        [clean_artist, clean_artist + '{'])

                if dbartist:
                    artistid = dbartist[0][0]

            if artistid:

                # This was previously using Artist, Album, Title with a SELECT LIKE ? and was not using an index
                # (Possible issue: https://stackoverflow.com/questions/37845854/python-sqlite3-not-using-index-with-like)
                # Now selects/updates using CleanName index (may have to revert if not working)

                # matching on CleanName should be enough, ensure it's the same artist just in case

                # Update tracks
                track = myDB.action(
                    'SELECT AlbumID, ArtistName FROM tracks WHERE CleanName = ? AND ArtistID = ?',
                    [clean_name, artistid]).fetchone()
                if track:
                    albumid = track['AlbumID']
                    myDB.action(
                        'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
                        [
                            song['Location'], song['BitRate'], song['Format'],
                            clean_name, artistid
                        ])

                # Update alltracks
                alltrack = myDB.action(
                    'SELECT AlbumID, ArtistName FROM alltracks WHERE CleanName = ? AND ArtistID = ?',
                    [clean_name, artistid]).fetchone()
                if alltrack:
                    albumid = alltrack['AlbumID']
                    myDB.action(
                        'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
                        [
                            song['Location'], song['BitRate'], song['Format'],
                            clean_name, artistid
                        ])

        # Update have
        controlValueDict2 = {'Location': song['Location']}
        if albumid:
            newValueDict2 = {'Matched': albumid}
        else:
            newValueDict2 = {'Matched': "Failed"}
        myDB.upsert("have", newValueDict2, controlValueDict2)

        # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])

    logger.info('Completed matching tracks from directory: %s' %
                dir.decode(headphones.SYS_ENCODING, 'replace'))

    if not append or artistScan:
        logger.info('Updating scanned artist track counts')

        # Clean up the new artist list
        unique_artists = {}.fromkeys(new_artists).keys()

        # # Don't think we need to do this, check the db instead below
        #
        # # artist scan
        # if ArtistName:
        #     current_artists = [[ArtistName]]
        # # directory scan
        # else:
        #     current_artists = myDB.select('SELECT ArtistName, ArtistID FROM artists WHERE ArtistName IS NOT NULL')
        #
        # # There was a bug where artists with special characters (-,') would show up in new artists.
        #
        # # artist_list = scanned artists not in the db
        # artist_list = [
        #     x for x in unique_artists
        #     if helpers.clean_name(x).lower() not in [
        #         helpers.clean_name(y[0]).lower()
        #         for y in current_artists
        #         ]
        #     ]
        #
        # # artists_checked = scanned artists that exist in the db
        # artists_checked = [
        #     x for x in unique_artists
        #     if helpers.clean_name(x).lower() in [
        #         helpers.clean_name(y[0]).lower()
        #         for y in current_artists
        #         ]
        #     ]

        new_artist_list = []

        for artist in unique_artists:

            if not artist:
                continue

            logger.info('Processing artist: %s' % artist)

            # check if artist is already in the db
            artist_lookup = "\"" + artist.replace("\"", "\"\"") + "\""

            try:
                dbartist = myDB.select(
                    'SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE '
                    + artist_lookup + '')
            except:
                dbartist = None
            if not dbartist:
                clean_artist = helpers.clean_name(artist)
                if clean_artist:
                    dbartist = myDB.select(
                        'SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
                        [clean_artist, clean_artist + '{'])
                    if not dbartist:
                        dbartist = myDB.select(
                            'SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
                            [clean_artist, clean_artist + '{'])

            # new artist not in db, add to list
            if not dbartist:
                new_artist_list.append(artist)
            else:

                # artist in db, update have track counts
                artistid = dbartist[0][0]

                # Have tracks are selected from tracks table and not all tracks because of duplicates
                # We update the track count upon an album switch to compliment this

                # havetracks = (
                #     len(myDB.select(
                #         'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
                #         [artist])) + len(myDB.select(
                #             'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
                #             [artist]))
                # )

                try:
                    havetracks = (len(
                        myDB.select(
                            'SELECT ArtistID From tracks WHERE ArtistID = ? AND Location IS NOT NULL',
                            [artistid])
                    ) + len(
                        myDB.select(
                            'SELECT ArtistName FROM have WHERE ArtistName LIKE '
                            + artist_lookup + ' AND Matched = "Failed"')))
                except Exception as e:
                    logger.warn('Error updating counts for artist: %s: %s' %
                                (artist, e))

                # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
                # (can fix by getting rid of second len statement)

                if havetracks:
                    myDB.action(
                        'UPDATE artists SET HaveTracks = ? WHERE ArtistID = ?',
                        [havetracks, artistid])

                    # Update albums to downloaded
                    update_album_status(ArtistID=artistid)

        logger.info('Found %i new artists' % len(new_artist_list))

        # Add scanned artists not in the db
        if new_artist_list:
            if headphones.CONFIG.AUTO_ADD_ARTISTS:
                logger.info('Importing %i new artists' % len(new_artist_list))
                importer.artistlist_to_mbids(new_artist_list)
            else:
                logger.info(
                    'To add these artists, go to Manage->Manage New Artists')
                # myDB.action('DELETE from newartists')
                for artist in new_artist_list:
                    myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)',
                                [artist])

        if headphones.CONFIG.DETECT_BITRATE and bitrates:
            headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len(
                bitrates) / 1000

    else:
        # If we're appending a new album to the database, update the artists total track counts
        logger.info('Updating artist track counts')

        artist_lookup = "\"" + ArtistName.replace("\"", "\"\"") + "\""
        try:
            havetracks = len(
                myDB.select(
                    'SELECT ArtistID FROM tracks WHERE ArtistID = ? AND Location IS NOT NULL',
                    [ArtistID])
            ) + len(
                myDB.select(
                    'SELECT ArtistName FROM have WHERE ArtistName LIKE ' +
                    artist_lookup + ' AND Matched = "Failed"'))
        except Exception as e:
            logger.warn('Error updating counts for artist: %s: %s' %
                        (ArtistName, e))

        if havetracks:
            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?',
                        [havetracks, ArtistID])

    # Moved above to call for each artist
    # if not append:
    #     update_album_status()

    if not append and not artistScan:
        lastfm.getSimilar()

    if ArtistName:
        logger.info('Scanning complete for artist: %s', ArtistName)
    else:
        logger.info('Library scan complete')
Пример #6
0
def addReleaseById(rid, rgid=None):
    myDB = db.DBConnection()

    # Create minimum info upfront if added from searchresults
    status = ''
    if rgid:
        dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid])
        if not dbalbum:
            status = 'Loading'
            controlValueDict = {"AlbumID": rgid}
            newValueDict = {"AlbumTitle": rgid,
                            "ArtistName": status,
                            "Status": status}
            myDB.upsert("albums", newValueDict, controlValueDict)
            time.sleep(1)

    rgid = None
    artistid = None
    release_dict = None
    results = myDB.select(
        "SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1",
        [rid])
    for result in results:
        rgid = result['ReleaseGroupID']
        artistid = result['ArtistID']
        logger.debug(
            "Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid)
    if not rgid:
        # didn't find it in the cache, get the information from MB
        logger.debug(
            "Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID")
        try:
            release_dict = mb.getRelease(rid)
        except Exception as e:
            logger.info('Unable to get release information for Release %s: %s', rid, e)
            if status == 'Loading':
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
            return
        if not release_dict:
            logger.info('Unable to get release information for Release %s: no dict', rid)
            if status == 'Loading':
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
            return

        rgid = release_dict['rgid']
        artistid = release_dict['artist_id']

    # we don't want to make more calls to MB here unless we have to, could be happening quite a lot
    rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid])

    # make sure the artist exists since I don't know what happens later if it doesn't
    artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid])

    if not artist_exists and release_dict:
        if release_dict['artist_name'].startswith('The '):
            sortname = release_dict['artist_name'][4:]
        else:
            sortname = release_dict['artist_name']

        logger.info(
            u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
        controlValueDict = {"ArtistID": release_dict['artist_id']}
        newValueDict = {"ArtistName": release_dict['artist_name'],
                        "ArtistSortName": sortname,
                        "DateAdded": helpers.today(),
                        "Status": "Paused"}

        if headphones.CONFIG.INCLUDE_EXTRAS:
            newValueDict['IncludeExtras'] = 1
            newValueDict['Extras'] = headphones.CONFIG.EXTRAS

        if 'title' in release_dict:
            newValueDict['LatestAlbum'] = release_dict['title']
        elif 'rg_title' in release_dict:
            newValueDict['LatestAlbum'] = release_dict['rg_title']

        if 'date' in release_dict:
            newValueDict['ReleaseDate'] = release_dict['date']

        if rgid:
            newValueDict['AlbumID'] = rgid

        myDB.upsert("artists", newValueDict, controlValueDict)

    elif not artist_exists and not release_dict:
        logger.error(
            "Artist does not exist in the database and did not get a valid response from MB. Skipping release.")
        if status == 'Loading':
            myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
        return

    if not rg_exists and release_dict or status == 'Loading' and release_dict:  # it should never be the case that we have an rg and not the artist
        # but if it is this will fail
        logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
        controlValueDict = {"AlbumID": rgid}
        if status != 'Loading':
            status = 'Wanted'

        newValueDict = {"ArtistID": release_dict['artist_id'],
                        "ReleaseID": rgid,
                        "ArtistName": release_dict['artist_name'],
                        "AlbumTitle": release_dict['title'] if 'title' in release_dict else
                        release_dict['rg_title'],
                        "AlbumASIN": release_dict['asin'],
                        "ReleaseDate": release_dict['date'],
                        "DateAdded": helpers.today(),
                        "Status": status,
                        "Type": release_dict['rg_type'],
                        "ReleaseID": rid
                        }

        myDB.upsert("albums", newValueDict, controlValueDict)

        # keep a local cache of these so that external programs that are adding releasesByID don't hammer MB
        myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']])

        for track in release_dict['tracks']:
            cleanname = helpers.clean_name(
                release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title'])

            controlValueDict = {"TrackID": track['id'],
                                "AlbumID": rgid}
            newValueDict = {"ArtistID": release_dict['artist_id'],
                            "ArtistName": release_dict['artist_name'],
                            "AlbumTitle": release_dict['rg_title'],
                            "AlbumASIN": release_dict['asin'],
                            "TrackTitle": track['title'],
                            "TrackDuration": track['duration'],
                            "TrackNumber": track['number'],
                            "CleanName": cleanname
                            }

            match = myDB.action(
                'SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?',
                [cleanname]).fetchone()

            if not match:
                match = myDB.action(
                    'SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                    [release_dict['artist_name'], release_dict['rg_title'],
                     track['title']]).fetchone()

                # if not match:
                # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()

            if match:
                newValueDict['Location'] = match['Location']
                newValueDict['BitRate'] = match['BitRate']
                newValueDict['Format'] = match['Format']
                # myDB.action('DELETE from have WHERE Location=?', [match['Location']])

                # If the album has been scanned before adding the release it will be unmatched, update to matched
                if match['Matched'] == 'Failed':
                    myDB.action('UPDATE have SET Matched=? WHERE Location=?',
                                (release_dict['rgid'], match['Location']))

            myDB.upsert("tracks", newValueDict, controlValueDict)

        # Reset status
        if status == 'Loading':
            controlValueDict = {"AlbumID": rgid}
            if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED:
                newValueDict = {"Status": "Wanted"}
            else:
                newValueDict = {"Status": "Skipped"}
            myDB.upsert("albums", newValueDict, controlValueDict)

        # Start a search for the album
        if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED:
            import searcher
            searcher.searchforalbum(rgid, False)

    elif not rg_exists and not release_dict:
        logger.error(
            "ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.")
        if status == 'Loading':
            myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
        return
    else:
        logger.info('Release ' + str(rid) + " already exists in the database!")
Пример #7
0
def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
    # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
    from headphones import cache

    # Can't add various artists - throws an error from MB
    if artistid in blacklisted_special_artists:
        logger.warn('Cannot import blocked special purpose artist with id' + artistid)
        return

    # We'll use this to see if we should update the 'LastUpdated' time stamp
    errors = False

    myDB = db.DBConnection()

    # Delete from blacklist if it's on there
    myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])

    # We need the current minimal info in the database instantly
    # so we don't throw a 500 error when we redirect to the artistPage
    controlValueDict = {"ArtistID": artistid}

    # Don't replace a known artist name with an "Artist ID" placeholder
    dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone()

    # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
    if not dbartist:
        newValueDict = {"ArtistName": "Artist ID: %s" % (artistid),
                        "Status": "Loading",
                        "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS,
                        "Extras": headphones.CONFIG.EXTRAS}
        if type == "series":
            newValueDict['Type'] = "series"
    else:
        newValueDict = {"Status": "Loading"}
        if dbartist["Type"] == "series":
            type = "series"

    myDB.upsert("artists", newValueDict, controlValueDict)

    if type == "series":
        artist = mb.getSeries(artistid)
    else:
        artist = mb.getArtist(artistid, extrasonly)

    if artist and artist.get('artist_name') in blacklisted_special_artist_names:
        logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name'))
        myDB.action('DELETE from artists WHERE ArtistID=?', [artistid])
        # in case it's already in the db
        myDB.action('DELETE from albums WHERE ArtistID=?', [artistid])
        myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid])
        return

    if not artist:
        logger.warn("Error fetching artist info. ID: " + artistid)
        if dbartist is None:
            newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid),
                            "Status": "Active"}
        else:
            newValueDict = {"Status": "Active"}
        myDB.upsert("artists", newValueDict, controlValueDict)
        return

    if artist['artist_name'].startswith('The '):
        sortname = artist['artist_name'][4:]
    else:
        sortname = artist['artist_name']

    logger.info(u"Now adding/updating: " + artist['artist_name'])
    controlValueDict = {"ArtistID": artistid}
    newValueDict = {"ArtistName": artist['artist_name'],
                    "ArtistSortName": sortname,
                    "DateAdded": helpers.today(),
                    "Status": "Loading"}

    myDB.upsert("artists", newValueDict, controlValueDict)

    # See if we need to grab extras. Artist specific extras take precedence
    # over global option. Global options are set when adding a new artist
    try:
        db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?',
                                [artistid]).fetchone()
        includeExtras = db_artist['IncludeExtras']
    except IndexError:
        includeExtras = False

    # Clean all references to release group in dB that are no longer referenced
    # from the musicbrainz refresh
    group_list = []
    force_repackage = 0

    # Don't nuke the database if there's a MusicBrainz error
    if len(artist['releasegroups']) != 0:
        for groups in artist['releasegroups']:
            group_list.append(groups['id'])
        if not extrasonly:
            remove_missing_groups_from_albums = myDB.select(
                "SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
        else:
            remove_missing_groups_from_albums = myDB.select(
                'SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"',
                [artistid])
        for items in remove_missing_groups_from_albums:
            if items['AlbumID'] not in group_list:
                # Remove all from albums/tracks that aren't in release groups
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']])
                myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']])
                myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']])
                myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']])
                myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']])
                logger.info(
                    "[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (
                    artist['artist_name'], items['AlbumID']))
                if not extrasonly:
                    force_repackage = 1
    else:
        if not extrasonly:
            logger.info(
                "[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" %
                artist['artist_name'])

    # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
    album_searches = []

    for rg in artist['releasegroups']:
        al_title = rg['title']
        today = helpers.today()
        rgid = rg['id']
        skip_log = 0
        # Make a user configurable variable to skip update of albums with release dates older than this date (in days)
        pause_delta = headphones.CONFIG.MB_IGNORE_AGE

        rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()

        if not forcefull:
            new_release_group = False

            try:
                check_release_date = rg_exists['ReleaseDate']
            except TypeError:
                check_release_date = None
                new_release_group = True

            if new_release_group:
                logger.info("[%s] Now adding: %s (New Release Group)" % (
                artist['artist_name'], rg['title']))
                new_releases = mb.get_new_releases(rgid, includeExtras)

            else:
                if check_release_date is None or check_release_date == u"None":
                    logger.info("[%s] Now updating: %s (No Release Date)" % (
                    artist['artist_name'], rg['title']))
                    new_releases = mb.get_new_releases(rgid, includeExtras, True)
                else:
                    if len(check_release_date) == 10:
                        release_date = check_release_date
                    elif len(check_release_date) == 7:
                        release_date = check_release_date + "-31"
                    elif len(check_release_date) == 4:
                        release_date = check_release_date + "-12-31"
                    else:
                        release_date = today
                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
                        logger.info("[%s] Now updating: %s (Release Date <%s Days)",
                                    artist['artist_name'], rg['title'], pause_delta)
                        new_releases = mb.get_new_releases(rgid, includeExtras, True)
                    else:
                        logger.info("[%s] Skipping: %s (Release Date >%s Days)",
                                    artist['artist_name'], rg['title'], pause_delta)
                        skip_log = 1
                        new_releases = 0

            if force_repackage == 1:
                new_releases = -1
                logger.info('[%s] Forcing repackage of %s (Release Group Removed)',
                            artist['artist_name'], al_title)
            else:
                new_releases = new_releases
        else:
            logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'],
                        rg['title'])
            new_releases = mb.get_new_releases(rgid, includeExtras, forcefull)

        if new_releases != 0:
            # Dump existing hybrid release since we're repackaging/replacing it
            myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']])
            myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rg['id']])

            # This will be used later to build a hybrid release
            fullreleaselist = []
            # Search for releases within a release group
            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?",
                                               [rg['id']])

            # Build the dictionary for the fullreleaselist
            for items in find_hybrid_releases:
                if items['ReleaseID'] != rg[
                    'id']:  # don't include hybrid information, since that's what we're replacing
                    hybrid_release_id = items['ReleaseID']
                    newValueDict = {"ArtistID": items['ArtistID'],
                                    "ArtistName": items['ArtistName'],
                                    "AlbumTitle": items['AlbumTitle'],
                                    "AlbumID": items['AlbumID'],
                                    "AlbumASIN": items['AlbumASIN'],
                                    "ReleaseDate": items['ReleaseDate'],
                                    "Type": items['Type'],
                                    "ReleaseCountry": items['ReleaseCountry'],
                                    "ReleaseFormat": items['ReleaseFormat']
                                    }
                    find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?",
                                                     [hybrid_release_id])
                    totalTracks = 1
                    hybrid_track_array = []
                    for hybrid_tracks in find_hybrid_tracks:
                        hybrid_track_array.append({
                            'number': hybrid_tracks['TrackNumber'],
                            'title': hybrid_tracks['TrackTitle'],
                            'id': hybrid_tracks['TrackID'],
                            # 'url':           hybrid_tracks['TrackURL'],
                            'duration': hybrid_tracks['TrackDuration']
                        })
                        totalTracks += 1
                    newValueDict['ReleaseID'] = hybrid_release_id
                    newValueDict['Tracks'] = hybrid_track_array
                    fullreleaselist.append(newValueDict)

            # Basically just do the same thing again for the hybrid release
            # This may end up being called with an empty fullreleaselist
            try:
                hybridrelease = getHybridRelease(fullreleaselist)
                logger.info('[%s] Packaging %s releases into hybrid title' % (
                artist['artist_name'], rg['title']))
            except Exception as e:
                errors = True
                logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (
                artist['artist_name'], rg['title'], e))
                continue

            # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
            # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
            # The hybrid won't have a country or a format
            controlValueDict = {"ReleaseID": rg['id']}

            newValueDict = {"ArtistID": artistid,
                            "ArtistName": artist['artist_name'],
                            "AlbumTitle": rg['title'],
                            "AlbumID": rg['id'],
                            "AlbumASIN": hybridrelease['AlbumASIN'],
                            "ReleaseDate": hybridrelease['ReleaseDate'],
                            "Type": rg['type']
                            }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            for track in hybridrelease['Tracks']:

                cleanname = helpers.clean_name(
                    artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])

                controlValueDict = {"TrackID": track['id'],
                                    "ReleaseID": rg['id']}

                newValueDict = {"ArtistID": artistid,
                                "ArtistName": artist['artist_name'],
                                "AlbumTitle": rg['title'],
                                "AlbumASIN": hybridrelease['AlbumASIN'],
                                "AlbumID": rg['id'],
                                "TrackTitle": track['title'],
                                "TrackDuration": track['duration'],
                                "TrackNumber": track['number'],
                                "CleanName": cleanname
                                }

                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?',
                                    [cleanname]).fetchone()

                if not match:
                    match = myDB.action(
                        'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                        [artist['artist_name'], rg['title'], track['title']]).fetchone()
                    # if not match:
                    # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                if match:
                    newValueDict['Location'] = match['Location']
                    newValueDict['BitRate'] = match['BitRate']
                    newValueDict['Format'] = match['Format']
                    # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                    myDB.action('UPDATE have SET Matched=? WHERE Location=?',
                                (rg['id'], match['Location']))

                myDB.upsert("alltracks", newValueDict, controlValueDict)

            # Delete matched tracks from the have table
            # myDB.action('DELETE from have WHERE Matched="True"')

            # If there's no release in the main albums tables, add the default (hybrid)
            # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
            # check if the album already exists
            if not rg_exists:
                releaseid = rg['id']
            else:
                releaseid = rg_exists['ReleaseID']
                if not releaseid:
                    releaseid = rg['id']

            album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone()

            controlValueDict = {"AlbumID": rg['id']}

            newValueDict = {"ArtistID": album['ArtistID'],
                            "ArtistName": album['ArtistName'],
                            "AlbumTitle": album['AlbumTitle'],
                            "ReleaseID": album['ReleaseID'],
                            "AlbumASIN": album['AlbumASIN'],
                            "ReleaseDate": album['ReleaseDate'],
                            "Type": album['Type'],
                            "ReleaseCountry": album['ReleaseCountry'],
                            "ReleaseFormat": album['ReleaseFormat']
                            }

            if rg_exists:
                newValueDict['DateAdded'] = rg_exists['DateAdded']
                newValueDict['Status'] = rg_exists['Status']

            else:
                today = helpers.today()

                newValueDict['DateAdded'] = today

                if headphones.CONFIG.AUTOWANT_ALL:
                    newValueDict['Status'] = "Wanted"
                elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING:
                    newValueDict['Status'] = "Wanted"
                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
                # The first test just makes sure we have year-month-day
                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(
                        today) - helpers.get_age(
                        album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
                    newValueDict['Status'] = "Wanted"
                else:
                    newValueDict['Status'] = "Skipped"

            myDB.upsert("albums", newValueDict, controlValueDict)

            tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?',
                                 [releaseid]).fetchall()

            # This is used to see how many tracks you have from an album - to
            # mark it as downloaded. Default is 80%, can be set in config as
            # ALBUM_COMPLETION_PCT
            total_track_count = len(tracks)

            if total_track_count == 0:
                logger.warning("Total track count is zero for Release ID " +
                               "'%s', skipping.", releaseid)
                continue

            for track in tracks:
                controlValueDict = {"TrackID": track['TrackID'],
                                    "AlbumID": rg['id']}

                newValueDict = {"ArtistID": track['ArtistID'],
                                "ArtistName": track['ArtistName'],
                                "AlbumTitle": track['AlbumTitle'],
                                "AlbumASIN": track['AlbumASIN'],
                                "ReleaseID": track['ReleaseID'],
                                "TrackTitle": track['TrackTitle'],
                                "TrackDuration": track['TrackDuration'],
                                "TrackNumber": track['TrackNumber'],
                                "CleanName": track['CleanName'],
                                "Location": track['Location'],
                                "Format": track['Format'],
                                "BitRate": track['BitRate']
                                }

                myDB.upsert("tracks", newValueDict, controlValueDict)

            # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
            have_track_count = len(
                myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL',
                            [rg['id']]))
            marked_as_downloaded = False

            if rg_exists:
                if rg_exists['Status'] == 'Skipped' and (
                    (have_track_count / float(total_track_count)) >= (
                    headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?',
                                ['Downloaded', rg['id']])
                    marked_as_downloaded = True
            else:
                if (have_track_count / float(total_track_count)) >= (
                    headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0):
                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?',
                                ['Downloaded', rg['id']])
                    marked_as_downloaded = True

            logger.info(
                u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
            cache.getThumb(AlbumID=rg['id'])

            # Start a search for the album if it's new, hasn't been marked as
            # downloaded and autowant_all is selected. This search is deferred,
            # in case the search failes and the rest of the import will halt.
            if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL:
                album_searches.append(rg['id'])
        else:
            if skip_log == 0:
                logger.info(u"[%s] No new releases, so no changes made to %s" % (
                artist['artist_name'], rg['title']))

    time.sleep(3)
    finalize_update(artistid, artist['artist_name'], errors)

    logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
    cache.getThumb(ArtistID=artistid)

    logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name'])
    metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])

    if errors:
        logger.info(
            "[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (
            artist['artist_name'], artist['artist_name']))
    else:
        myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']])
        logger.info(u"Updating complete for: %s" % artist['artist_name'])

    # Start searching for newly added albums
    if album_searches:
        from headphones import searcher
        logger.info("Start searching for %d albums.", len(album_searches))

        for album_search in album_searches:
            searcher.searchforalbum(albumid=album_search)
Пример #8
0
def addReleaseById(rid, rgid=None):
    myDB = db.DBConnection()

    # Create minimum info upfront if added from searchresults
    status = ''
    if rgid:
        dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid])
        if not dbalbum:
            status = 'Loading'
            controlValueDict = {"AlbumID": rgid}
            newValueDict = {
                "AlbumTitle": rgid,
                "ArtistName": status,
                "Status": status
            }
            myDB.upsert("albums", newValueDict, controlValueDict)
            time.sleep(1)

    rgid = None
    artistid = None
    release_dict = None
    results = myDB.select(
        "SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1",
        [rid])
    for result in results:
        rgid = result['ReleaseGroupID']
        artistid = result['ArtistID']
        logger.debug(
            "Found a cached releaseid : releasegroupid relationship: " + rid +
            " : " + rgid)
    if not rgid:
        # didn't find it in the cache, get the information from MB
        logger.debug("Didn't find releaseID " + rid +
                     " in the cache. Looking up its ReleaseGroupID")
        try:
            release_dict = mb.getRelease(rid)
        except Exception as e:
            logger.info('Unable to get release information for Release %s: %s',
                        rid, e)
            if status == 'Loading':
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
            return
        if not release_dict:
            logger.info(
                'Unable to get release information for Release %s: no dict',
                rid)
            if status == 'Loading':
                myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
            return

        rgid = release_dict['rgid']
        artistid = release_dict['artist_id']

    # we don't want to make more calls to MB here unless we have to, could be happening quite a lot
    rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid])

    # make sure the artist exists since I don't know what happens later if it doesn't
    artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?",
                                [artistid])

    if not artist_exists and release_dict:
        if release_dict['artist_name'].startswith('The '):
            sortname = release_dict['artist_name'][4:]
        else:
            sortname = release_dict['artist_name']

        logger.info(u"Now manually adding: " + release_dict['artist_name'] +
                    " - with status Paused")
        controlValueDict = {"ArtistID": release_dict['artist_id']}
        newValueDict = {
            "ArtistName": release_dict['artist_name'],
            "ArtistSortName": sortname,
            "DateAdded": helpers.today(),
            "Status": "Paused"
        }

        if headphones.CONFIG.INCLUDE_EXTRAS:
            newValueDict['IncludeExtras'] = 1
            newValueDict['Extras'] = headphones.CONFIG.EXTRAS

        if 'title' in release_dict:
            newValueDict['LatestAlbum'] = release_dict['title']
        elif 'rg_title' in release_dict:
            newValueDict['LatestAlbum'] = release_dict['rg_title']

        if 'date' in release_dict:
            newValueDict['ReleaseDate'] = release_dict['date']

        if rgid:
            newValueDict['AlbumID'] = rgid

        myDB.upsert("artists", newValueDict, controlValueDict)

    elif not artist_exists and not release_dict:
        logger.error(
            "Artist does not exist in the database and did not get a valid response from MB. Skipping release."
        )
        if status == 'Loading':
            myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
        return

    if not rg_exists and release_dict or status == 'Loading' and release_dict:  # it should never be the case that we have an rg and not the artist
        # but if it is this will fail
        logger.info(u"Now adding-by-id album (" + release_dict['title'] +
                    ") from id: " + rgid)
        controlValueDict = {"AlbumID": rgid}
        if status != 'Loading':
            status = 'Wanted'

        newValueDict = {
            "ArtistID":
            release_dict['artist_id'],
            "ReleaseID":
            rgid,
            "ArtistName":
            release_dict['artist_name'],
            "AlbumTitle":
            release_dict['title']
            if 'title' in release_dict else release_dict['rg_title'],
            "AlbumASIN":
            release_dict['asin'],
            "ReleaseDate":
            release_dict['date'],
            "DateAdded":
            helpers.today(),
            "Status":
            status,
            "Type":
            release_dict['rg_type'],
            "ReleaseID":
            rid
        }

        myDB.upsert("albums", newValueDict, controlValueDict)

        # keep a local cache of these so that external programs that are adding releasesByID don't hammer MB
        myDB.action('INSERT INTO releases VALUES( ?, ?)',
                    [rid, release_dict['rgid']])

        for track in release_dict['tracks']:
            cleanname = helpers.clean_name(release_dict['artist_name'] + ' ' +
                                           release_dict['rg_title'] + ' ' +
                                           track['title'])

            controlValueDict = {"TrackID": track['id'], "AlbumID": rgid}
            newValueDict = {
                "ArtistID": release_dict['artist_id'],
                "ArtistName": release_dict['artist_name'],
                "AlbumTitle": release_dict['rg_title'],
                "AlbumASIN": release_dict['asin'],
                "TrackTitle": track['title'],
                "TrackDuration": track['duration'],
                "TrackNumber": track['number'],
                "CleanName": cleanname
            }

            match = myDB.action(
                'SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?',
                [cleanname]).fetchone()

            if not match:
                match = myDB.action(
                    'SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                    [
                        release_dict['artist_name'], release_dict['rg_title'],
                        track['title']
                    ]).fetchone()

                # if not match:
                # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()

            if match:
                newValueDict['Location'] = match['Location']
                newValueDict['BitRate'] = match['BitRate']
                newValueDict['Format'] = match['Format']
                # myDB.action('DELETE from have WHERE Location=?', [match['Location']])

                # If the album has been scanned before adding the release it will be unmatched, update to matched
                if match['Matched'] == 'Failed':
                    myDB.action('UPDATE have SET Matched=? WHERE Location=?',
                                (release_dict['rgid'], match['Location']))

            myDB.upsert("tracks", newValueDict, controlValueDict)

        # Reset status
        if status == 'Loading':
            controlValueDict = {"AlbumID": rgid}
            if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED:
                newValueDict = {"Status": "Wanted"}
            else:
                newValueDict = {"Status": "Skipped"}
            myDB.upsert("albums", newValueDict, controlValueDict)

        # Start a search for the album
        if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED:
            import searcher
            searcher.searchforalbum(rgid, False)

    elif not rg_exists and not release_dict:
        logger.error(
            "ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release."
        )
        if status == 'Loading':
            myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
        return
    else:
        logger.info('Release ' + str(rid) + " already exists in the database!")
Пример #9
0
def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
    # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
    from headphones import cache

    # Can't add various artists - throws an error from MB
    if artistid in blacklisted_special_artists:
        logger.warn('Cannot import blocked special purpose artist with id' +
                    artistid)
        return

    # We'll use this to see if we should update the 'LastUpdated' time stamp
    errors = False

    myDB = db.DBConnection()

    # Delete from blacklist if it's on there
    myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid])

    # We need the current minimal info in the database instantly
    # so we don't throw a 500 error when we redirect to the artistPage
    controlValueDict = {"ArtistID": artistid}

    # Don't replace a known artist name with an "Artist ID" placeholder
    dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?',
                           [artistid]).fetchone()

    # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch
    if not dbartist:
        newValueDict = {
            "ArtistName": "Artist ID: %s" % (artistid),
            "Status": "Loading",
            "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS,
            "Extras": headphones.CONFIG.EXTRAS
        }
        if type == "series":
            newValueDict['Type'] = "series"
    else:
        newValueDict = {"Status": "Loading"}
        if dbartist["Type"] == "series":
            type = "series"

    myDB.upsert("artists", newValueDict, controlValueDict)

    if type == "series":
        artist = mb.getSeries(artistid)
    else:
        artist = mb.getArtist(artistid, extrasonly)

    if artist and artist.get(
            'artist_name') in blacklisted_special_artist_names:
        logger.warn('Cannot import blocked special purpose artist: %s' %
                    artist.get('artist_name'))
        myDB.action('DELETE from artists WHERE ArtistID=?', [artistid])
        # in case it's already in the db
        myDB.action('DELETE from albums WHERE ArtistID=?', [artistid])
        myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid])
        return

    if not artist:
        logger.warn("Error fetching artist info. ID: " + artistid)
        if dbartist is None:
            newValueDict = {
                "ArtistName":
                "Fetch failed, try refreshing. (%s)" % (artistid),
                "Status": "Active"
            }
        else:
            newValueDict = {"Status": "Active"}
        myDB.upsert("artists", newValueDict, controlValueDict)
        return

    if artist['artist_name'].startswith('The '):
        sortname = artist['artist_name'][4:]
    else:
        sortname = artist['artist_name']

    logger.info(u"Now adding/updating: " + artist['artist_name'])
    controlValueDict = {"ArtistID": artistid}
    newValueDict = {
        "ArtistName": artist['artist_name'],
        "ArtistSortName": sortname,
        "DateAdded": helpers.today(),
        "Status": "Loading"
    }

    myDB.upsert("artists", newValueDict, controlValueDict)

    # See if we need to grab extras. Artist specific extras take precedence
    # over global option. Global options are set when adding a new artist
    try:
        db_artist = myDB.action(
            'SELECT IncludeExtras, Extras from artists WHERE ArtistID=?',
            [artistid]).fetchone()
        includeExtras = db_artist['IncludeExtras']
    except IndexError:
        includeExtras = False

    # Clean all references to release group in dB that are no longer referenced
    # from the musicbrainz refresh
    group_list = []
    force_repackage = 0

    # Don't nuke the database if there's a MusicBrainz error
    if len(artist['releasegroups']) != 0:
        for groups in artist['releasegroups']:
            group_list.append(groups['id'])
        if not extrasonly:
            remove_missing_groups_from_albums = myDB.select(
                "SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
        else:
            remove_missing_groups_from_albums = myDB.select(
                'SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"',
                [artistid])
        for items in remove_missing_groups_from_albums:
            if items['AlbumID'] not in group_list:
                # Remove all from albums/tracks that aren't in release groups
                myDB.action("DELETE FROM albums WHERE AlbumID=?",
                            [items['AlbumID']])
                myDB.action("DELETE FROM allalbums WHERE AlbumID=?",
                            [items['AlbumID']])
                myDB.action("DELETE FROM tracks WHERE AlbumID=?",
                            [items['AlbumID']])
                myDB.action("DELETE FROM alltracks WHERE AlbumID=?",
                            [items['AlbumID']])
                myDB.action('DELETE from releases WHERE ReleaseGroupID=?',
                            [items['AlbumID']])
                logger.info(
                    "[%s] Removing all references to release group %s to reflect MusicBrainz refresh"
                    % (artist['artist_name'], items['AlbumID']))
                if not extrasonly:
                    force_repackage = 1
    else:
        if not extrasonly:
            logger.info(
                "[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category"
                % artist['artist_name'])

    # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
    album_searches = []

    for rg in artist['releasegroups']:
        al_title = rg['title']
        today = helpers.today()
        rgid = rg['id']
        skip_log = 0
        # Make a user configurable variable to skip update of albums with release dates older than this date (in days)
        pause_delta = headphones.CONFIG.MB_IGNORE_AGE

        rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?",
                                [rg['id']]).fetchone()

        if not forcefull:
            new_release_group = False

            try:
                check_release_date = rg_exists['ReleaseDate']
            except TypeError:
                check_release_date = None
                new_release_group = True

            if new_release_group:
                logger.info("[%s] Now adding: %s (New Release Group)" %
                            (artist['artist_name'], rg['title']))
                new_releases = mb.get_new_releases(rgid, includeExtras)

            else:
                if check_release_date is None or check_release_date == u"None":
                    if headphones.CONFIG.MB_IGNORE_AGE_MISSING is not 1:
                        logger.info("[%s] Now updating: %s (No Release Date)" %
                                    (artist['artist_name'], rg['title']))
                        new_releases = mb.get_new_releases(
                            rgid, includeExtras, True)
                    else:
                        logger.info(
                            "[%s] Skipping update of: %s (No Release Date)" %
                            (artist['artist_name'], rg['title']))
                        new_releases = 0
                else:
                    if len(check_release_date) == 10:
                        release_date = check_release_date
                    elif len(check_release_date) == 7:
                        release_date = check_release_date + "-31"
                    elif len(check_release_date) == 4:
                        release_date = check_release_date + "-12-31"
                    else:
                        release_date = today
                    if helpers.get_age(today) - helpers.get_age(
                            release_date) < pause_delta:
                        logger.info(
                            "[%s] Now updating: %s (Release Date <%s Days)",
                            artist['artist_name'], rg['title'], pause_delta)
                        new_releases = mb.get_new_releases(
                            rgid, includeExtras, True)
                    else:
                        logger.info(
                            "[%s] Skipping: %s (Release Date >%s Days)",
                            artist['artist_name'], rg['title'], pause_delta)
                        skip_log = 1
                        new_releases = 0

            if force_repackage == 1:
                new_releases = -1
                logger.info(
                    '[%s] Forcing repackage of %s (Release Group Removed)',
                    artist['artist_name'], al_title)
            else:
                new_releases = new_releases
        else:
            logger.info("[%s] Now adding/updating: %s (Comprehensive Force)",
                        artist['artist_name'], rg['title'])
            new_releases = mb.get_new_releases(rgid, includeExtras, forcefull)

        if new_releases != 0:
            # Dump existing hybrid release since we're repackaging/replacing it
            myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']])
            myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']])
            myDB.action('DELETE from releases WHERE ReleaseGroupID=?',
                        [rg['id']])

            # This will be used later to build a hybrid release
            fullreleaselist = []
            # Search for releases within a release group
            find_hybrid_releases = myDB.action(
                "SELECT * from allalbums WHERE AlbumID=?", [rg['id']])

            # Build the dictionary for the fullreleaselist
            for items in find_hybrid_releases:
                # don't include hybrid information, since that's what we're replacing
                if items['ReleaseID'] != rg['id']:
                    hybrid_release_id = items['ReleaseID']
                    newValueDict = {
                        "ArtistID": items['ArtistID'],
                        "ArtistName": items['ArtistName'],
                        "AlbumTitle": items['AlbumTitle'],
                        "AlbumID": items['AlbumID'],
                        "AlbumASIN": items['AlbumASIN'],
                        "ReleaseDate": items['ReleaseDate'],
                        "Type": items['Type'],
                        "ReleaseCountry": items['ReleaseCountry'],
                        "ReleaseFormat": items['ReleaseFormat']
                    }
                    find_hybrid_tracks = myDB.action(
                        "SELECT * from alltracks WHERE ReleaseID=?",
                        [hybrid_release_id])
                    totalTracks = 1
                    hybrid_track_array = []
                    for hybrid_tracks in find_hybrid_tracks:
                        hybrid_track_array.append({
                            'number':
                            hybrid_tracks['TrackNumber'],
                            'title':
                            hybrid_tracks['TrackTitle'],
                            'id':
                            hybrid_tracks['TrackID'],
                            # 'url':           hybrid_tracks['TrackURL'],
                            'duration':
                            hybrid_tracks['TrackDuration']
                        })
                        totalTracks += 1
                    newValueDict['ReleaseID'] = hybrid_release_id
                    newValueDict['Tracks'] = hybrid_track_array
                    fullreleaselist.append(newValueDict)

            # Basically just do the same thing again for the hybrid release
            # This may end up being called with an empty fullreleaselist
            try:
                hybridrelease = getHybridRelease(fullreleaselist)
                logger.info('[%s] Packaging %s releases into hybrid title' %
                            (artist['artist_name'], rg['title']))
            except Exception as e:
                errors = True
                logger.warn(
                    '[%s] Unable to get hybrid release information for %s: %s'
                    % (artist['artist_name'], rg['title'], e))
                continue

            # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
            # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
            # The hybrid won't have a country or a format
            controlValueDict = {"ReleaseID": rg['id']}

            newValueDict = {
                "ArtistID": artistid,
                "ArtistName": artist['artist_name'],
                "AlbumTitle": rg['title'],
                "AlbumID": rg['id'],
                "AlbumASIN": hybridrelease['AlbumASIN'],
                "ReleaseDate": hybridrelease['ReleaseDate'],
                "Type": rg['type']
            }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            for track in hybridrelease['Tracks']:

                cleanname = helpers.clean_name(artist['artist_name'] + ' ' +
                                               rg['title'] + ' ' +
                                               track['title'])

                controlValueDict = {
                    "TrackID": track['id'],
                    "ReleaseID": rg['id']
                }

                newValueDict = {
                    "ArtistID": artistid,
                    "ArtistName": artist['artist_name'],
                    "AlbumTitle": rg['title'],
                    "AlbumASIN": hybridrelease['AlbumASIN'],
                    "AlbumID": rg['id'],
                    "TrackTitle": track['title'],
                    "TrackDuration": track['duration'],
                    "TrackNumber": track['number'],
                    "CleanName": cleanname
                }

                match = myDB.action(
                    'SELECT Location, BitRate, Format from have WHERE CleanName=?',
                    [cleanname]).fetchone()

                if not match:
                    match = myDB.action(
                        'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                        [artist['artist_name'], rg['title'], track['title']
                         ]).fetchone()
                    # if not match:
                    # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                if match:
                    newValueDict['Location'] = match['Location']
                    newValueDict['BitRate'] = match['BitRate']
                    newValueDict['Format'] = match['Format']
                    # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                    myDB.action('UPDATE have SET Matched=? WHERE Location=?',
                                (rg['id'], match['Location']))

                myDB.upsert("alltracks", newValueDict, controlValueDict)

            # Delete matched tracks from the have table
            # myDB.action('DELETE from have WHERE Matched="True"')

            # If there's no release in the main albums tables, add the default (hybrid)
            # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
            # check if the album already exists
            if not rg_exists:
                releaseid = rg['id']
            else:
                releaseid = rg_exists['ReleaseID']
                if not releaseid:
                    releaseid = rg['id']

            album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?',
                                [releaseid]).fetchone()

            controlValueDict = {"AlbumID": rg['id']}

            newValueDict = {
                "ArtistID": album['ArtistID'],
                "ArtistName": album['ArtistName'],
                "AlbumTitle": album['AlbumTitle'],
                "ReleaseID": album['ReleaseID'],
                "AlbumASIN": album['AlbumASIN'],
                "ReleaseDate": album['ReleaseDate'],
                "Type": album['Type'],
                "ReleaseCountry": album['ReleaseCountry'],
                "ReleaseFormat": album['ReleaseFormat']
            }

            if rg_exists:
                newValueDict['DateAdded'] = rg_exists['DateAdded']
                newValueDict['Status'] = rg_exists['Status']

            else:
                today = helpers.today()

                newValueDict['DateAdded'] = today

                if headphones.CONFIG.AUTOWANT_ALL:
                    newValueDict['Status'] = "Wanted"
                elif album[
                        'ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING:
                    newValueDict['Status'] = "Wanted"
                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
                # The first test just makes sure we have year-month-day
                elif helpers.get_age(
                        album['ReleaseDate']
                ) and helpers.get_age(today) - helpers.get_age(
                        album['ReleaseDate']
                ) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
                    newValueDict['Status'] = "Wanted"
                else:
                    newValueDict['Status'] = "Skipped"

            myDB.upsert("albums", newValueDict, controlValueDict)

            tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?',
                                 [releaseid]).fetchall()

            # This is used to see how many tracks you have from an album - to
            # mark it as downloaded. Default is 80%, can be set in config as
            # ALBUM_COMPLETION_PCT
            total_track_count = len(tracks)

            if total_track_count == 0:
                logger.warning(
                    "Total track count is zero for Release ID " +
                    "'%s', skipping.", releaseid)
                continue

            for track in tracks:
                controlValueDict = {
                    "TrackID": track['TrackID'],
                    "AlbumID": rg['id']
                }

                newValueDict = {
                    "ArtistID": track['ArtistID'],
                    "ArtistName": track['ArtistName'],
                    "AlbumTitle": track['AlbumTitle'],
                    "AlbumASIN": track['AlbumASIN'],
                    "ReleaseID": track['ReleaseID'],
                    "TrackTitle": track['TrackTitle'],
                    "TrackDuration": track['TrackDuration'],
                    "TrackNumber": track['TrackNumber'],
                    "CleanName": track['CleanName'],
                    "Location": track['Location'],
                    "Format": track['Format'],
                    "BitRate": track['BitRate']
                }

                myDB.upsert("tracks", newValueDict, controlValueDict)

            # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
            have_track_count = len(
                myDB.select(
                    'SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL',
                    [rg['id']]))
            marked_as_downloaded = False

            if rg_exists:
                if rg_exists['Status'] == 'Skipped' and (
                    (have_track_count / float(total_track_count)) >=
                    (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?',
                                ['Downloaded', rg['id']])
                    marked_as_downloaded = True
            else:
                if (have_track_count / float(total_track_count)) >= (
                        headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0):
                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?',
                                ['Downloaded', rg['id']])
                    marked_as_downloaded = True

            logger.info(u"[%s] Seeing if we need album art for %s" %
                        (artist['artist_name'], rg['title']))
            cache.getThumb(AlbumID=rg['id'])

            # Start a search for the album if it's new, hasn't been marked as
            # downloaded and autowant_all is selected. This search is deferred,
            # in case the search failes and the rest of the import will halt.
            if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL:
                album_searches.append(rg['id'])
        else:
            if skip_log == 0:
                logger.info(u"[%s] No new releases, so no changes made to %s" %
                            (artist['artist_name'], rg['title']))

    time.sleep(3)
    finalize_update(artistid, artist['artist_name'], errors)

    logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
    cache.getThumb(ArtistID=artistid)

    logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name'])
    metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])

    if errors:
        logger.info(
            "[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database"
            % (artist['artist_name'], artist['artist_name']))
    else:
        myDB.action('DELETE FROM newartists WHERE ArtistName = ?',
                    [artist['artist_name']])
        logger.info(u"Updating complete for: %s" % artist['artist_name'])

    # Start searching for newly added albums
    if album_searches:
        from headphones import searcher
        logger.info("Start searching for %d albums.", len(album_searches))

        for album_search in album_searches:
            searcher.searchforalbum(albumid=album_search)
Пример #10
0
def get_new_releases(rgid, includeExtras=False, forcefull=False):
    myDB = db.DBConnection()
    results = []

    release_status = "official"
    if includeExtras and not headphones.CONFIG.OFFICIAL_RELEASES_ONLY:
        release_status = []

    try:
        limit = 100
        newResults = None
        while newResults is None or len(newResults) >= limit:
            with mb_lock:
                newResults = musicbrainzngs.browse_releases(
                    release_group=rgid,
                    includes=[
                        'artist-credits', 'labels', 'recordings',
                        'release-groups', 'media'
                    ],
                    release_status=release_status,
                    limit=limit,
                    offset=len(results))
            if 'release-list' not in newResults:
                break  # may want to raise an exception here instead ?
            newResults = newResults['release-list']
            results += newResults

    except musicbrainzngs.WebServiceError as e:
        logger.warn(
            'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)'
            % (rgid, str(e)))
        mb_lock.snooze(5)
        return False

    if not results or len(results) == 0:
        return False

    # Clean all references to releases in dB that are no longer referenced in musicbrainz
    release_list = []
    force_repackage1 = 0
    if len(results) != 0:
        for release_mark in results:
            release_list.append(unicode(release_mark['id']))
            release_title = release_mark['title']
        remove_missing_releases = myDB.action(
            "SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid])
        if remove_missing_releases:
            for items in remove_missing_releases:
                if items['ReleaseID'] not in release_list and items[
                        'ReleaseID'] != rgid:
                    # Remove all from albums/tracks that aren't in release
                    myDB.action("DELETE FROM albums WHERE ReleaseID=?",
                                [items['ReleaseID']])
                    myDB.action("DELETE FROM tracks WHERE ReleaseID=?",
                                [items['ReleaseID']])
                    myDB.action("DELETE FROM allalbums WHERE ReleaseID=?",
                                [items['ReleaseID']])
                    myDB.action("DELETE FROM alltracks WHERE ReleaseID=?",
                                [items['ReleaseID']])
                    logger.info(
                        "Removing all references to release %s to reflect MusicBrainz"
                        % items['ReleaseID'])
                    force_repackage1 = 1
    else:
        logger.info(
            "There was either an error pulling data from MusicBrainz or there might not be any releases for this category"
        )

    num_new_releases = 0

    for releasedata in results:

        release = {}
        rel_id_check = releasedata['id']
        album_checker = myDB.action(
            'SELECT * from allalbums WHERE ReleaseID=?',
            [rel_id_check]).fetchone()
        if not album_checker or forcefull:
            # DELETE all references to this release since we're updating it anyway.
            myDB.action('DELETE from allalbums WHERE ReleaseID=?',
                        [rel_id_check])
            myDB.action('DELETE from alltracks WHERE ReleaseID=?',
                        [rel_id_check])
            release['AlbumTitle'] = unicode(releasedata['title'])
            release['AlbumID'] = unicode(rgid)
            release['AlbumASIN'] = unicode(
                releasedata['asin']) if 'asin' in releasedata else None
            release['ReleaseDate'] = unicode(
                releasedata['date']) if 'date' in releasedata else None
            release['ReleaseID'] = releasedata['id']
            if 'release-group' not in releasedata:
                raise Exception(
                    'No release group associated with release id ' +
                    releasedata['id'] + ' album id' + rgid)
            release['Type'] = unicode(releasedata['release-group']['type'])

            if release[
                    'Type'] == 'Album' and 'secondary-type-list' in releasedata[
                        'release-group']:
                secondary_type = unicode(
                    releasedata['release-group']['secondary-type-list'][0])
                if secondary_type != release['Type']:
                    release['Type'] = secondary_type

            # making the assumption that the most important artist will be first in the list
            if 'artist-credit' in releasedata:
                release['ArtistID'] = unicode(
                    releasedata['artist-credit'][0]['artist']['id'])
                release['ArtistName'] = unicode(
                    releasedata['artist-credit-phrase'])
            else:
                logger.warn('Release ' + releasedata['id'] +
                            ' has no Artists associated.')
                return False

            release['ReleaseCountry'] = unicode(
                releasedata['country']
            ) if 'country' in releasedata else u'Unknown'
            # assuming that the list will contain media and that the format will be consistent
            try:
                additional_medium = ''
                for position in releasedata['medium-list']:
                    if position['format'] == releasedata['medium-list'][0][
                            'format']:
                        medium_count = int(position['position'])
                    else:
                        additional_medium = additional_medium + ' + ' + position[
                            'format']
                if medium_count == 1:
                    disc_number = ''
                else:
                    disc_number = str(medium_count) + 'x'
                packaged_medium = disc_number + releasedata['medium-list'][0][
                    'format'] + additional_medium
                release['ReleaseFormat'] = unicode(packaged_medium)
            except:
                release['ReleaseFormat'] = u'Unknown'

            release['Tracks'] = getTracksFromRelease(releasedata)

            # What we're doing here now is first updating the allalbums & alltracks table to the most
            # current info, then moving the appropriate release into the album table and its associated
            # tracks into the tracks table
            controlValueDict = {"ReleaseID": release['ReleaseID']}

            newValueDict = {
                "ArtistID": release['ArtistID'],
                "ArtistName": release['ArtistName'],
                "AlbumTitle": release['AlbumTitle'],
                "AlbumID": release['AlbumID'],
                "AlbumASIN": release['AlbumASIN'],
                "ReleaseDate": release['ReleaseDate'],
                "Type": release['Type'],
                "ReleaseCountry": release['ReleaseCountry'],
                "ReleaseFormat": release['ReleaseFormat']
            }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            for track in release['Tracks']:

                cleanname = helpers.clean_name(release['ArtistName'] + ' ' +
                                               release['AlbumTitle'] + ' ' +
                                               track['title'])

                controlValueDict = {
                    "TrackID": track['id'],
                    "ReleaseID": release['ReleaseID']
                }

                newValueDict = {
                    "ArtistID": release['ArtistID'],
                    "ArtistName": release['ArtistName'],
                    "AlbumTitle": release['AlbumTitle'],
                    "AlbumID": release['AlbumID'],
                    "AlbumASIN": release['AlbumASIN'],
                    "TrackTitle": track['title'],
                    "TrackDuration": track['duration'],
                    "TrackNumber": track['number'],
                    "CleanName": cleanname
                }

                match = myDB.action(
                    'SELECT Location, BitRate, Format from have WHERE CleanName=?',
                    [cleanname]).fetchone()

                if not match:
                    match = myDB.action(
                        'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                        [
                            release['ArtistName'], release['AlbumTitle'],
                            track['title']
                        ]).fetchone()
                    # if not match:
                    # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                if match:
                    newValueDict['Location'] = match['Location']
                    newValueDict['BitRate'] = match['BitRate']
                    newValueDict['Format'] = match['Format']
                    # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                    myDB.action('UPDATE have SET Matched=? WHERE Location=?',
                                (release['AlbumID'], match['Location']))

                myDB.upsert("alltracks", newValueDict, controlValueDict)
            num_new_releases = num_new_releases + 1
            if album_checker:
                logger.info('[%s] Existing release %s (%s) updated' %
                            (release['ArtistName'], release['AlbumTitle'],
                             rel_id_check))
            else:
                logger.info('[%s] New release %s (%s) added' %
                            (release['ArtistName'], release['AlbumTitle'],
                             rel_id_check))
        if force_repackage1 == 1:
            num_new_releases = -1
            logger.info(
                '[%s] Forcing repackage of %s, since dB releases have been removed'
                % (release['ArtistName'], release_title))
        else:
            num_new_releases = num_new_releases

    return num_new_releases
Пример #11
0
def get_new_releases(rgid, includeExtras=False, forcefull=False):
    myDB = db.DBConnection()
    results = []

    release_status = "official"
    if includeExtras and not headphones.CONFIG.OFFICIAL_RELEASES_ONLY:
        release_status = []

    try:
        limit = 100
        newResults = None
        while newResults is None or len(newResults) >= limit:
            with mb_lock:
                newResults = musicbrainzngs.browse_releases(
                    release_group=rgid,
                    includes=["artist-credits", "labels", "recordings", "release-groups", "media"],
                    release_status=release_status,
                    limit=limit,
                    offset=len(results),
                )
            if "release-list" not in newResults:
                break  # may want to raise an exception here instead ?
            newResults = newResults["release-list"]
            results += newResults

    except musicbrainzngs.WebServiceError as e:
        logger.warn(
            'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))
        )
        mb_lock.snooze(5)
        return False

    if not results or len(results) == 0:
        return False

    # Clean all references to releases in dB that are no longer referenced in musicbrainz
    release_list = []
    force_repackage1 = 0
    if len(results) != 0:
        for release_mark in results:
            release_list.append(unicode(release_mark["id"]))
            release_title = release_mark["title"]
        remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid])
        if remove_missing_releases:
            for items in remove_missing_releases:
                if items["ReleaseID"] not in release_list and items["ReleaseID"] != rgid:
                    # Remove all from albums/tracks that aren't in release
                    myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items["ReleaseID"]])
                    myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items["ReleaseID"]])
                    myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items["ReleaseID"]])
                    myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items["ReleaseID"]])
                    logger.info("Removing all references to release %s to reflect MusicBrainz" % items["ReleaseID"])
                    force_repackage1 = 1
    else:
        logger.info(
            "There was either an error pulling data from MusicBrainz or there might not be any releases for this category"
        )

    num_new_releases = 0

    for releasedata in results:

        release = {}
        rel_id_check = releasedata["id"]
        album_checker = myDB.action("SELECT * from allalbums WHERE ReleaseID=?", [rel_id_check]).fetchone()
        if not album_checker or forcefull:
            # DELETE all references to this release since we're updating it anyway.
            myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rel_id_check])
            myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rel_id_check])
            release["AlbumTitle"] = unicode(releasedata["title"])
            release["AlbumID"] = unicode(rgid)
            release["AlbumASIN"] = unicode(releasedata["asin"]) if "asin" in releasedata else None
            release["ReleaseDate"] = unicode(releasedata["date"]) if "date" in releasedata else None
            release["ReleaseID"] = releasedata["id"]
            if "release-group" not in releasedata:
                raise Exception("No release group associated with release id " + releasedata["id"] + " album id" + rgid)
            release["Type"] = unicode(releasedata["release-group"]["type"])

            if release["Type"] == "Album" and "secondary-type-list" in releasedata["release-group"]:
                secondary_type = unicode(releasedata["release-group"]["secondary-type-list"][0])
                if secondary_type != release["Type"]:
                    release["Type"] = secondary_type

            # making the assumption that the most important artist will be first in the list
            if "artist-credit" in releasedata:
                release["ArtistID"] = unicode(releasedata["artist-credit"][0]["artist"]["id"])
                release["ArtistName"] = unicode(releasedata["artist-credit-phrase"])
            else:
                logger.warn("Release " + releasedata["id"] + " has no Artists associated.")
                return False

            release["ReleaseCountry"] = unicode(releasedata["country"]) if "country" in releasedata else u"Unknown"
            # assuming that the list will contain media and that the format will be consistent
            try:
                additional_medium = ""
                for position in releasedata["medium-list"]:
                    if position["format"] == releasedata["medium-list"][0]["format"]:
                        medium_count = int(position["position"])
                    else:
                        additional_medium = additional_medium + " + " + position["format"]
                if medium_count == 1:
                    disc_number = ""
                else:
                    disc_number = str(medium_count) + "x"
                packaged_medium = disc_number + releasedata["medium-list"][0]["format"] + additional_medium
                release["ReleaseFormat"] = unicode(packaged_medium)
            except:
                release["ReleaseFormat"] = u"Unknown"

            release["Tracks"] = getTracksFromRelease(releasedata)

            # What we're doing here now is first updating the allalbums & alltracks table to the most
            # current info, then moving the appropriate release into the album table and its associated
            # tracks into the tracks table
            controlValueDict = {"ReleaseID": release["ReleaseID"]}

            newValueDict = {
                "ArtistID": release["ArtistID"],
                "ArtistName": release["ArtistName"],
                "AlbumTitle": release["AlbumTitle"],
                "AlbumID": release["AlbumID"],
                "AlbumASIN": release["AlbumASIN"],
                "ReleaseDate": release["ReleaseDate"],
                "Type": release["Type"],
                "ReleaseCountry": release["ReleaseCountry"],
                "ReleaseFormat": release["ReleaseFormat"],
            }

            myDB.upsert("allalbums", newValueDict, controlValueDict)

            for track in release["Tracks"]:

                cleanname = helpers.clean_name(
                    release["ArtistName"] + " " + release["AlbumTitle"] + " " + track["title"]
                )

                controlValueDict = {"TrackID": track["id"], "ReleaseID": release["ReleaseID"]}

                newValueDict = {
                    "ArtistID": release["ArtistID"],
                    "ArtistName": release["ArtistName"],
                    "AlbumTitle": release["AlbumTitle"],
                    "AlbumID": release["AlbumID"],
                    "AlbumASIN": release["AlbumASIN"],
                    "TrackTitle": track["title"],
                    "TrackDuration": track["duration"],
                    "TrackNumber": track["number"],
                    "CleanName": cleanname,
                }

                match = myDB.action(
                    "SELECT Location, BitRate, Format from have WHERE CleanName=?", [cleanname]
                ).fetchone()

                if not match:
                    match = myDB.action(
                        "SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?",
                        [release["ArtistName"], release["AlbumTitle"], track["title"]],
                    ).fetchone()
                    # if not match:
                    # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                if match:
                    newValueDict["Location"] = match["Location"]
                    newValueDict["BitRate"] = match["BitRate"]
                    newValueDict["Format"] = match["Format"]
                    # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                    myDB.action("UPDATE have SET Matched=? WHERE Location=?", (release["AlbumID"], match["Location"]))

                myDB.upsert("alltracks", newValueDict, controlValueDict)
            num_new_releases = num_new_releases + 1
            if album_checker:
                logger.info(
                    "[%s] Existing release %s (%s) updated"
                    % (release["ArtistName"], release["AlbumTitle"], rel_id_check)
                )
            else:
                logger.info(
                    "[%s] New release %s (%s) added" % (release["ArtistName"], release["AlbumTitle"], rel_id_check)
                )
        if force_repackage1 == 1:
            num_new_releases = -1
            logger.info(
                "[%s] Forcing repackage of %s, since dB releases have been removed"
                % (release["ArtistName"], release_title)
            )
        else:
            num_new_releases = num_new_releases

    return num_new_releases
Пример #12
0
def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                cron=False, artistScan=False):
    if cron and not headphones.CONFIG.LIBRARYSCAN:
        return

    if not dir:
        if not headphones.CONFIG.MUSIC_DIR:
            return
        else:
            dir = headphones.CONFIG.MUSIC_DIR

    # If we're appending a dir, it's coming from the post processor which is
    # already bytestring
    if not append or artistScan:
        dir = dir.encode(headphones.SYS_ENCODING)

    if not os.path.isdir(dir):
        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING,
                                                                           'replace'))
        return

    myDB = db.DBConnection()
    new_artists = []

    logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))

    if not append:
        # Clean up bad filepaths
        tracks = myDB.select(
            'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL')

        for track in tracks:
            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
            if not os.path.isfile(encoded_track_string):
                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                            [None, None, None, track['Location']])
                myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                            [None, None, None, track['Location']])

        del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have')

        for track in del_have_tracks:
            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
            if not os.path.isfile(encoded_track_string):
                if track['ArtistName']:
                    # Make sure deleted files get accounted for when updating artist track counts
                    new_artists.append(track['ArtistName'])
                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
                logger.info(
                    'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(
                        headphones.SYS_ENCODING, 'replace'))

    bitrates = []
    song_list = []
    latest_subdirectory = []

    new_song_count = 0
    file_count = 0

    for r, d, f in helpers.walk_directory(dir):
        # Filter paths based on config. Note that these methods work directly
        # on the inputs
        helpers.path_filter_patterns(d, headphones.CONFIG.IGNORED_FOLDERS, r)
        helpers.path_filter_patterns(f, headphones.CONFIG.IGNORED_FILES, r)

        for files in f:
            # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
            if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
                subdirectory = r.replace(dir, '')
                latest_subdirectory.append(subdirectory)

                if file_count == 0 and r.replace(dir, '') != '':
                    logger.info("[%s] Now scanning subdirectory %s" % (
                        dir.decode(headphones.SYS_ENCODING, 'replace'),
                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
                elif latest_subdirectory[file_count] != latest_subdirectory[
                            file_count - 1] and file_count != 0:
                    logger.info("[%s] Now scanning subdirectory %s" % (
                        dir.decode(headphones.SYS_ENCODING, 'replace'),
                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))

                song = os.path.join(r, files)

                # We need the unicode path to use for logging, inserting into database
                unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace')

                # Try to read the metadata
                try:
                    f = MediaFile(song)
                except (FileTypeError, UnreadableFileError):
                    logger.warning(
                        "Cannot read media file '%s', skipping. It may be corrupted or not a media file.",
                        unicode_song_path)
                    continue
                except IOError:
                    logger.warning("Cannnot read media file '%s', skipping. Does the file exists?",
                                   unicode_song_path)
                    continue

                # Grab the bitrates for the auto detect bit rate option
                if f.bitrate:
                    bitrates.append(f.bitrate)

                # Use the album artist over the artist if available
                if f.albumartist:
                    f_artist = f.albumartist
                elif f.artist:
                    f_artist = f.artist
                else:
                    f_artist = None

                # Add the song to our song list -
                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)

                if f_artist and f.album and f.title:
                    CleanName = helpers.clean_name(f_artist + ' ' + f.album + ' ' + f.title)
                else:
                    CleanName = None

                controlValueDict = {'Location': unicode_song_path}

                newValueDict = {'TrackID': f.mb_trackid,
                                # 'ReleaseID' : f.mb_albumid,
                                'ArtistName': f_artist,
                                'AlbumTitle': f.album,
                                'TrackNumber': f.track,
                                'TrackLength': f.length,
                                'Genre': f.genre,
                                'Date': f.date,
                                'TrackTitle': f.title,
                                'BitRate': f.bitrate,
                                'Format': f.format,
                                'CleanName': CleanName
                                }

                # song_list.append(song_dict)
                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?",
                                               [unicode_song_path]).fetchone()
                # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
                if not check_exist_song:
                    # This is a new track
                    if f_artist:
                        new_artists.append(f_artist)
                    myDB.upsert("have", newValueDict, controlValueDict)
                    new_song_count += 1
                else:
                    if check_exist_song['ArtistName'] != f_artist or check_exist_song[
                            'AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
                        # Important track metadata has been modified, need to run matcher again
                        if f_artist and f_artist != check_exist_song['ArtistName']:
                            new_artists.append(f_artist)
                        elif f_artist and f_artist == check_exist_song['ArtistName'] and \
                                        check_exist_song['Matched'] != "Ignored":
                            new_artists.append(f_artist)
                        else:
                            continue

                        newValueDict['Matched'] = None
                        myDB.upsert("have", newValueDict, controlValueDict)
                        myDB.action(
                            'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                            [None, None, None, unicode_song_path])
                        myDB.action(
                            'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
                            [None, None, None, unicode_song_path])
                        new_song_count += 1
                    else:
                        # This track information hasn't changed
                        if f_artist and check_exist_song['Matched'] != "Ignored":
                            new_artists.append(f_artist)

                file_count += 1

    # Now we start track matching
    logger.info("%s new/modified songs found and added to the database" % new_song_count)
    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
                            [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"])
    total_number_of_songs = \
        myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
                    [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0]
    logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(
        headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")

    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches

    # song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])

    # We'll use this to give a % completion, just because the track matching might take a while
    song_count = 0
    latest_artist = []
    last_completion_percentage = 0

    for song in song_list:

        latest_artist.append(song['ArtistName'])
        if song_count == 0:
            logger.info("Now matching songs by %s" % song['ArtistName'])
        elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0:
            logger.info("Now matching songs by %s" % song['ArtistName'])

        song_count += 1
        completion_percentage = math.floor(float(song_count) / total_number_of_songs * 1000) / 10

        if completion_percentage >= (last_completion_percentage + 10):
            logger.info("Track matching is " + str(completion_percentage) + "% complete")
            last_completion_percentage = completion_percentage

        # THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED.  WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT
        # EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME.  ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION,
        # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
        # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.

        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:

            track = myDB.action(
                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
            have_updated = False
            if track:
                controlValueDict = {'ArtistName': track['ArtistName'],
                                    'AlbumTitle': track['AlbumTitle'],
                                    'TrackTitle': track['TrackTitle']}
                newValueDict = {'Location': song['Location'],
                                'BitRate': song['BitRate'],
                                'Format': song['Format']}
                myDB.upsert("tracks", newValueDict, controlValueDict)

                controlValueDict2 = {'Location': song['Location']}
                newValueDict2 = {'Matched': track['AlbumID']}
                myDB.upsert("have", newValueDict2, controlValueDict2)
                have_updated = True
            else:
                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?',
                                    [song['CleanName']]).fetchone()
                if track:
                    controlValueDict = {'CleanName': track['CleanName']}
                    newValueDict = {'Location': song['Location'],
                                    'BitRate': song['BitRate'],
                                    'Format': song['Format']}
                    myDB.upsert("tracks", newValueDict, controlValueDict)

                    controlValueDict2 = {'Location': song['Location']}
                    newValueDict2 = {'Matched': track['AlbumID']}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
                    have_updated = True
                else:
                    controlValueDict2 = {'Location': song['Location']}
                    newValueDict2 = {'Matched': "Failed"}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
                    have_updated = True

            alltrack = myDB.action(
                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
            if alltrack:
                controlValueDict = {'ArtistName': alltrack['ArtistName'],
                                    'AlbumTitle': alltrack['AlbumTitle'],
                                    'TrackTitle': alltrack['TrackTitle']}
                newValueDict = {'Location': song['Location'],
                                'BitRate': song['BitRate'],
                                'Format': song['Format']}
                myDB.upsert("alltracks", newValueDict, controlValueDict)

                controlValueDict2 = {'Location': song['Location']}
                newValueDict2 = {'Matched': alltrack['AlbumID']}
                myDB.upsert("have", newValueDict2, controlValueDict2)
            else:
                alltrack = myDB.action(
                    'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?',
                    [song['CleanName']]).fetchone()
                if alltrack:
                    controlValueDict = {'CleanName': alltrack['CleanName']}
                    newValueDict = {'Location': song['Location'],
                                    'BitRate': song['BitRate'],
                                    'Format': song['Format']}
                    myDB.upsert("alltracks", newValueDict, controlValueDict)

                    controlValueDict2 = {'Location': song['Location']}
                    newValueDict2 = {'Matched': alltrack['AlbumID']}
                    myDB.upsert("have", newValueDict2, controlValueDict2)
                else:
                    # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks
                    if not have_updated:
                        controlValueDict2 = {'Location': song['Location']}
                        newValueDict2 = {'Matched': "Failed"}
                        myDB.upsert("have", newValueDict2, controlValueDict2)

        else:
            controlValueDict2 = {'Location': song['Location']}
            newValueDict2 = {'Matched': "Failed"}
            myDB.upsert("have", newValueDict2, controlValueDict2)

            # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])

    logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING,
                                                                            'replace'))

    if not append or artistScan:
        logger.info('Updating scanned artist track counts')

        # Clean up the new artist list
        unique_artists = {}.fromkeys(new_artists).keys()
        current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')

        # There was a bug where artists with special characters (-,') would show up in new artists.
        artist_list = [
            x for x in unique_artists
            if helpers.clean_name(x).lower() not in [
                helpers.clean_name(y[0]).lower()
                for y in current_artists
                ]
            ]
        artists_checked = [
            x for x in unique_artists
            if helpers.clean_name(x).lower() in [
                helpers.clean_name(y[0]).lower()
                for y in current_artists
                ]
            ]

        # Update track counts
        for artist in artists_checked:
            # Have tracks are selected from tracks table and not all tracks because of duplicates
            # We update the track count upon an album switch to compliment this
            havetracks = (
                len(myDB.select(
                    'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
                    [artist])) + len(myDB.select(
                        'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
                        [artist]))
            )
            # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
            # (can fix by getting rid of second len statement)
            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist])

        logger.info('Found %i new artists' % len(artist_list))

        if artist_list:
            if headphones.CONFIG.AUTO_ADD_ARTISTS:
                logger.info('Importing %i new artists' % len(artist_list))
                importer.artistlist_to_mbids(artist_list)
            else:
                logger.info('To add these artists, go to Manage->Manage New Artists')
                # myDB.action('DELETE from newartists')
                for artist in artist_list:
                    myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist])

        if headphones.CONFIG.DETECT_BITRATE and bitrates:
            headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000

    else:
        # If we're appending a new album to the database, update the artists total track counts
        logger.info('Updating artist track counts')

        havetracks = len(
            myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
                        [ArtistID])) + len(myDB.select(
                            'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
                            [ArtistName]))
        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])

    if not append:
        update_album_status()

    if not append and not artistScan:
        lastfm.getSimilar()

    logger.info('Library scan complete')