def run_queries(discs, result_list, language, fingerprint, mixed,
do_quick_match):
# Try a text-based match first.
(match1, albums1, arts1) = run_query_on_discs(discs, result_list, language,
fingerprint, mixed,
do_quick_match)
final_match = match1
# If the result looks shoddy, try with fingerprinting.
if albums1 > len(discs) or match1 < 75 or arts1 == 0:
Log('Not impressed, trying the other way (fingerprinting: %s)' %
(not fingerprint))
other_result_list = []
(match2, albums2,
arts2) = run_query_on_discs(discs, other_result_list, language,
not fingerprint, mixed, do_quick_match)
if match2 > match1 or (match2 == match1 and
(albums2 < albums1 or arts2 > arts1)):
Log('This way gave a better match, keeping.')
result_list[:] = other_result_list
final_match = match2
return final_match
def group_tracks_by_disc(query_list):
tracks_by_disc = defaultdict(list)
# See if we have multiple disks, first checking tags.
discs = set([t.disc for t in query_list if t.disc is not None])
if len(discs) > 1:
for t in query_list:
tracks_by_disc[t.disc].append(t)
return tracks_by_disc.values()
# Otherwise, let's sort by filename, and see if we have clusters of tracks.
sorted_tracks = sorted(query_list, key=lambda track: track.parts[0])
disc = 1
last_index = 0
for t in sorted_tracks:
if t.index < last_index:
disc += 1
if t.index != 1:
Log('Disc %d didn\'t start with first track, we won\'t use this method.'
% disc)
tracks_by_disc = defaultdict(list)
break
tracks_by_disc[disc].append(t)
last_index = t.index
if len(tracks_by_disc) > 1:
return tracks_by_disc.values()
# Otherwise, let's consider it a single disc.
return [query_list]
def improve_from_tag(existing, file, tag):
tags = None
try:
tags = mutagen.File(file, easy=True)
except:
Log('There was an exception thrown reading tags.')
if tags and tag in tags:
existing = tags[tag][0]
return toBytes(existing)
def run_query_on_discs(discs, result_list, language, fingerprint, mixed,
do_quick_match):
match1 = albums1 = total_tracks = 0
for tracks in discs:
(match, albums1, arts1) = lookup(tracks,
result_list,
language=language,
fingerprint=fingerprint,
mixed=mixed,
do_quick_match=do_quick_match)
total_tracks += len(tracks)
match1 += match * len(tracks)
match1 = match1 / float(total_tracks)
Log('Querying all discs generated %d albums and a total match of %d' %
(albums1, match1))
return (match1, albums1, arts1)
def lookup(query_list,
result_list,
language=None,
fingerprint=False,
mixed=False,
multiple=False,
do_quick_match=False):
# This shouldn't happen, but be safe.
if len(query_list) == 0:
return (0, 0, 0)
# See if input looks like a sane album
(sane_input_tracks, unique_input_albums,
input_discs) = compute_input_sanity(query_list)
# Build up the query with the contents of the query list.
args = ''
parts = {}
Log('Running Gracenote match on %d tracks with fingerprinting: %d and mixedContent: %d and multiple: %d'
% (len(query_list), fingerprint, mixed, multiple))
for i, track in enumerate(query_list):
# We need to pass at least a path and an identifier for each track that we know about.
args += '&tracks[%d].path=%s' % (i, quote(track.parts[0], ''))
args += '&tracks[%d].userData=%d' % (i, i)
# Keep track of the identifier -> part mapping so we can reassemble later.
parts[i] = track.parts[0]
if track.name:
args += '&tracks[%d].title=%s' % (
i, quote(toBytes(track.title or track.name), ''))
if track.artist and track.artist != 'Various Artists':
args += '&tracks[%d].artist=%s' % (
i, quote(toBytes(track.artist), ''))
if track.album_artist:
args += '&tracks[%d].albumArtist=%s' % (
i, quote(toBytes(track.album_artist), ''))
elif track.artist and track.artist != 'Various Artists':
args += '&tracks[%d].albumArtist=%s' % (
i, quote(toBytes(track.artist), ''))
if track.album and track.album != '[Unknown Album]':
args += '&tracks[%d].album=%s' % (i, quote(toBytes(track.album),
''))
if track.index:
args += '&tracks[%d].index=%s' % (i, track.index)
if track.disc:
args += '&tracks[%d].parentIndex=%s' % (i, track.disc)
Log(' - %s/%s - %s/%s - %s' %
(toBytes(track.artist), toBytes(track.album), toBytes(
track.disc), toBytes(track.index), toBytes(track.name)))
url = 'http://127.0.0.1:32400/services/gracenote/search?fingerprint=%d&mixedContent=%d&multiple=%d%s&lang=%s' % (
fingerprint, mixed, multiple, args, language)
try:
res = minidom.parse(urlopen(url))
except Exception, e:
Log('Error parsing Gracenote response: ' + str(e))
return (0, 0, 0)
def Scan(path,
files,
media_list,
subdirs,
language=None,
root=None,
respect_tags=False):
# Scan for audio files.
AudioFiles.Scan(path, files, media_list, subdirs, root)
root_str = root or ''
loc_str = os.path.join(root_str, path)
Log('Scanning: ' + loc_str)
Log('Files: ' + str(files))
Log('Subdirs: ' + str(subdirs))
# Look at the files and determine whether we can do a quick match (minimal tag parsing).
do_quick_match = True
mixed = False
# Make sure we're looking at a leaf directory (no audio files below here).
if len(subdirs) > 0:
Log('Found directories below this one; won\'t attempt quick matching.')
do_quick_match = False
if files:
# Make sure we're not sitting in the section root.
parent_path = os.path.split(files[0])[0]
if parent_path == root:
Log('File(s) are in section root; doing expensive matching with mixed content.'
)
do_quick_match = False
mixed = True
# Make sure we have reliable track indices for all files and there are no dupes.
tracks = {}
for f in files:
try:
index = re.search(r'^([0-9]{1,2})[^0-9].*',
os.path.split(f)[-1]).groups(0)[0]
except:
do_quick_match = False
Log('Couldn\'t find track indices in all filenames; doing expensive matching.'
)
break
if tracks.get(index):
do_quick_match = False
mixed = True
Log('Found duplicate track index: %s; doing expensive matching with mixed content.'
% index)
break
else:
tracks[index] = True
# Read the first track's tags to check for milti-disc and VA.
if do_quick_match:
disc = album_artist = None
try:
(artist, album, title, track, disc, album_artist,
compil) = AudioFiles.getInfoFromTag(files[0], language)
except:
Log('Exception reading tags from first file; doing expensive matching.'
)
do_quick_match = False
# Make sure we are on the first disc.
if disc is not None and disc > 1:
Log('Skipping quick match because of non-first disc.')
do_quick_match = False
# We want to read all the tags for VA albums to pick up track artists.
if album_artist is not None and album_artist == 'Various Artists':
Log('Skipping quick match for Various Artists album.')
do_quick_match = False
artist = None
album = None
if do_quick_match:
Log('Doing quick match')
# See if we have some consensus on artist/album by reading a few tags.
for i in range(3):
if i < len(files):
this_artist = this_album = tags = None
try:
tags = mutagen.File(files[i], easy=True)
except:
Log('There was an exception thrown reading tags.')
if tags:
# See if there's an album artist tag.
album_artist_tags = [
t for t in ['albumartist', 'TPE2', 'performer']
if t in tags
]
album_artist_tag = album_artist_tags[0] if len(
album_artist_tags) else None
this_artist = tags[album_artist_tag][
0] if album_artist_tag else tags['artist'][
0] if 'artist' in tags else None
this_album = tags['album'][
0] if 'album' in tags else None
if artist and artist != this_artist:
Log('Found different artists in tags (%s vs. %s); doing expensive matching.'
% (artist, this_artist))
do_quick_match = False
break
if album and album != this_album:
Log('Found different albums in tags (%s vs. %s); doing expensive matching.'
% (album, this_album))
do_quick_match = False
break
artist = this_artist
album = this_album
if not artist or not album:
Log('Couldn\'t determine unique artist or album from tags; doing expensive matching.'
)
do_quick_match = False
query_list = []
result_list = []
fingerprint = False
# Directory looks clean, let's build a query list directly from info gleaned from file names.
if do_quick_match:
Log('Building query list for quickmatch with artist: %s, album: %s'
% (artist, album))
# Determine if the artist and/or album appears in all filenames, since we'll want to strip these out for clean titles.
strip_artist = True if len([
f for f in files if artist.lower() in Unicodize(
os.path.basename(f), language).lower()
]) == len(files) else False
strip_album = True if len([
f for f in files if album.lower() in Unicodize(
os.path.basename(f), language).lower()
]) == len(files) else False
for f in files:
try:
filename = os.path.splitext(os.path.split(f)[1])[0]
(head, index, title) = re.split(r'^([0-9]{1,2})', filename)
# Replace underscores and dots with spaces.
title = re.sub(r'[_\. ]+', ' ', title)
# Things in parens seem to confuse Gracenote, so let's strip them out.
title = re.sub(r' ?\(.*\)', '', title)
# Remove artist name from title if it appears in all of them.
if strip_artist and len(files) > 2:
title = re.sub(r'(?i)' + artist, '', title)
# Remove album title from title if it appears in all of them.
if strip_album and len(files) > 2:
title = re.sub(r'(?i)' + album, '', title)
# Remove any remaining index-, artist-, and album-related cruft from the head of the track title.
title = re.sub(r'^[\W\-]+', '', title).strip()
# Last chance for artist or album prefix.
if not strip_artist and Unicodize(
title, language).lower().find(artist.lower()) == 0:
title = title[len(artist):]
if not strip_album and Unicodize(
title, language).lower().find(album.lower()) == 0:
title = title[len(album):]
t = Media.Track(artist=toBytes(artist),
album=toBytes(album),
title=toBytes(title),
index=int(index))
t.parts.append(f)
Log(' - Adding: %s - %s' % (index, title))
query_list.append(t)
except Exception as e:
Log('Error preparing tracks for quick matching: ' + str(e))
# Otherwise, let's do old school directory crawling and tag reading.
else:
AudioFiles.Process(path, files, media_list, subdirs, root)
query_list = list(media_list)
# Try as-is first (ask for everything at once).
discs = [query_list]
final_match = run_queries(discs, result_list, language, fingerprint,
mixed, do_quick_match)
# If the match was still shitty, and it looks like we have multiple discs, try splitting.
if final_match < 75:
discs = group_tracks_by_disc(query_list)
if len(discs) > 1:
Log('Result still looked bad, we will try splitting into separate per-disc queries.'
)
other_result_list = []
other_match = run_queries(discs, other_result_list, language,
fingerprint, mixed, do_quick_match)
if other_match > final_match:
Log('The split result was best, we will use it.')
result_list = other_result_list
final_match = other_match
# If we have a crappy match, don't use it.
if final_match < 50.0:
Log('That was terrible, let us not use it.')
result_list = []
# Finalize the results.
used_tags = False
del media_list[:]
if len(result_list) > 0:
# Gracenote results.
for result in result_list:
media_list.append(result)
else:
# We bailed during the GN lookup, fall back to tags.
used_tags = True
AudioFiles.Process(path, files, media_list, subdirs, root)
# If we wanted to respect tags, then make sure we used tags.
if not used_tags and respect_tags:
# Let's grab tag results, and then set GUIDs we found.
tag_media_list = []
AudioFiles.Process(path, files, tag_media_list, subdirs, root)
# Now suck GN data out.
path_map = {}
for track in media_list:
path_map[track.parts[0]] = track
for track in tag_media_list:
if track.parts[0] in path_map:
gn_track = path_map[track.parts[0]]
track.guid = gn_track.guid
track.album_guid = gn_track.album_guid
track.artist_guid = gn_track.artist_guid
track.album_thumb_url = gn_track.album_thumb_url
track.artist_thumb_url = gn_track.artist_thumb_url
# If the tags failed, fill in key data from Gracenote.
if track.album == '[Unknown Album]':
track.album = gn_track.album
if track.artist == '[Unknown Artist]':
track.artist = gn_track.artist
media_list[:] = tag_media_list
}
# Figure out the unique artists/albums/indexes.
unique_artists = len(
set([
t[1].getAttribute('grandparentTitle')
for t in matched_tracks.items()
]))
unique_albums = len(
set([t[1].getAttribute('parentTitle')
for t in matched_tracks.items()]))
unique_indices = len(
set([t[1].getAttribute('index') for t in matched_tracks.items()]))
if DEBUG:
Log('Raw track matches:')
for track in [match[1] for match in matched_tracks.items()]:
Log(' - %s / %s - %s/%s - %s' %
(track.getAttribute('grandparentTitle'),
track.getAttribute('parentTitle'),
track.getAttribute('parentIndex'),
track.getAttribute('index'), track.getAttribute('title')))
# Look through the results and determine some consensus metadata so we can do a better job of keeping rogue and
# unmatched tracks together. We're going to weight matches in the first third of the tracks twice as high, for
# cases in which matches come through for the last half of tracks.
#
sorted_items = sorted(
matched_tracks.items(),
key=lambda t: int(t[1].getAttribute('parentIndex') or 1) * 100 + int(t[
1].getAttribute('index') or -1))