def get_releases_for_recording(recording_id):
# Because there may be more than 25 releases for a recording, we should use
# the browse methods
# the browse release API endpoint doesn't let you retrieve tags for linked releases.
# Because of this, we only gather IDs here, and return them.
# we get artist, releasegroup, and tag information in `get_metadata_for_releases`
offset = 0
all_releases = []
releases = mb.browse_releases(recording=recording_id, offset=0)
total_releases = releases["release-count"]
all_releases += releases["release-list"]
offset += len(releases["release-list"])
while len(all_releases) < total_releases:
releases = mb.browse_releases(recording=recording_id, offset=offset)
all_releases += releases["release-list"]
offset += len(releases["release-list"])
release_ids = []
for r in all_releases:
release_ids.append(r["id"])
return release_ids
def test_browse_release(self):
artist = "47f67b22-affe-4fe1-9d25-853d69bc0ee3"
musicbrainzngs.browse_releases(artist=artist)
self.assertEqual(
"http://musicbrainz.org/ws/2/release/?artist=47f67b22-affe-4fe1-9d25-853d69bc0ee3",
self.opener.get_url())
musicbrainzngs.browse_releases(track_artist=artist)
self.assertEqual(
"http://musicbrainz.org/ws/2/release/?track_artist=47f67b22-affe-4fe1-9d25-853d69bc0ee3",
self.opener.get_url())
label = "713c4a95-6616-442b-9cf6-14e1ddfd5946"
musicbrainzngs.browse_releases(label=label)
self.assertEqual(
"http://musicbrainz.org/ws/2/release/?label=713c4a95-6616-442b-9cf6-14e1ddfd5946",
self.opener.get_url())
recording = "7484fcfd-1968-4401-a44d-d1edcc580518"
musicbrainzngs.browse_releases(recording=recording)
self.assertEqual(
"http://musicbrainz.org/ws/2/release/?recording=7484fcfd-1968-4401-a44d-d1edcc580518",
self.opener.get_url())
release_group = "1c1b54f7-e56a-3ce8-b62c-e45c378e7f76"
musicbrainzngs.browse_releases(release_group=release_group)
self.assertEqual(
"http://musicbrainz.org/ws/2/release/?release-group=1c1b54f7-e56a-3ce8-b62c-e45c378e7f76",
self.opener.get_url())
def get_artist_releases(artist_mbid):
limit = 100
offset = 0
releases = []
release_groups = []
page = 1
result = mbz.browse_releases(artist=artist_mbid,
limit=limit,
offset=offset,
includes=['release-groups', 'artist-credits'],
release_status=['official'])
releases += result['release-list']
if 'release-count' in result:
count = result['release-count']
while len(releases) < count:
offset += limit
page += 1
result = mbz.browse_releases(
artist=artist_mbid,
limit=limit,
offset=offset,
includes=['release-groups', 'artist-credits'],
release_status=['official'])
releases += result['release-list']
for release in releases:
if release not in release_groups:
release_groups.append(release['release-group'])
return {'status': 200, 'releases': release_groups}
def get_all_releases_from_artist(s_artist_id):
limit = 100
offset = 0
page = 1
# d_search_parameters = dict(artist=s_artist_id, release_type=["album"])
d_search_parameters = dict(artist=s_artist_id, limit=limit)
first_page = musicbrainzngs.browse_releases(**d_search_parameters)
page_releases = first_page['release-list']
yield from ({
"title": albums["title"],
"date": albums["date"]
} for albums in page_releases)
while len(page_releases) >= limit:
offset += limit
page += 1
print("fetching page number %d.." % page)
next_page = musicbrainzngs.browse_releases(offset=offset,
**d_search_parameters)
page_releases = next_page['release-list']
yield from ({
"title": albums["title"],
"date": albums["date"]
} for albums in page_releases
if "date" in albums and "title" in albums)
def get_releases_for_release_group(release_group_id, includes=RELEASE_INCLUDES):
with musicbrainz_lock:
logger.debug('Fetching release for rgid {}'.format(release_group_id))
search_results = musicbrainzngs.browse_releases(release_group=release_group_id,
release_type='album', includes=includes)['release-list']
return [album_info(release) for release in search_results]
def _parse_recordings(recordings):
_recording = None
_results = []
for recording in recordings:
if MusicEntityGenerator._validate_score(_recording, recording):
_recording = recording
recording_releases = musicbrainzngs.browse_releases(
recording=recording['id'],
includes=['labels', 'release-groups']
)
# Find all valid releases (Official releases)
for rel in recording_releases['release-list']:
if MusicEntityGenerator._valid_release(rel):
_results.append((rel, rel['label-info-list']))
# Clean up recording data
if 'release-list' in recording:
recording.pop('release-list')
else:
break
return _recording, _results
def get_artist_tracks_from_musicbrianz_api(artist):
search_results = mb.search_artists(artist)
result = search_results['artist-list'][0]
genre = Artist.get_genre_from_musicbrainz_tag_list(result['tag-list'])
for album_dict in mb.browse_releases(result['id'],
includes=['recordings'
])['release-list']:
album = Album.objects.create(name=album_dict['title'],
artist=artist,
slug=slugify(album_dict['title']))
for track_dict in album_dict['medium-list'][0]['track-list']:
track = Track.objects.create(album=album,
name=track_dict['recording']['title'],
track_number=track_dict['position'],
slug=slugify(
track_dict['recording']['title']))
Artist.objects.create(track=track,
artist=artist,
genre=genre,
slug=slugify(artist))
return Artist.objects.filter(artist=artist)
def album_record_label(
release_group_musicbrainz_id) -> 'record_label_musicbrainz_id':
"""Extracts the record label associated with an album.
Taken the releases belonging to the release-group, the first valid record label is considered
Returns:
str -- record label musicbrainz id
"""
if release_group_musicbrainz_id is not None:
releases = mz.browse_releases(
release_group=release_group_musicbrainz_id['value'],
includes=['labels'])
for r in releases['release-list']:
label_list = r['label-info-list']
if len(label_list) > 0:
for l in label_list:
try:
if l['label']['name'] != '[no label]':
return {'value': l['label']['id']}
except KeyError:
continue
logging.getLogger('root.features').warning(
f"Release {r['id']} has no associated record label")
return None
def get_release_list(artist_str):
""" Gets a release list from musicbrains API. """
username = '******'
password = '******'
musicbrainzngs.set_useragent(username, password)
artist_list = musicbrainzngs.search_artists(
artist=artist_str)['artist-list']
artist = sorted(artist_list,
reverse=True,
key=lambda artist: int(artist['ext:score']))[0]
artist_id = artist['id']
limit = 100
offset = 0
release_list = []
release_count = 1
while offset < release_count:
print 'Requesting tracks {0} - {1}'.format(str(offset),
str(offset + limit))
result = musicbrainzngs.browse_releases(artist=artist_id,
release_status=['official'],
release_type=['album'],
includes=['recordings'],
limit=limit,
offset=offset)
release_count = result['release-count']
release_list += result['release-list']
offset += limit
return release_list
def artist_recorded_label(
artist_musicbrainz_id) -> 'record_label_musicbrainz_id':
"""Harvest the list of record lables the artist has published records with
Arguments:
artist_musicbrainz_id {str} --
Returns:
list -- Record lables ids
"""
releases = mz.browse_releases(artist=artist_musicbrainz_id['value'],
includes=['labels'])
labels = set()
for r in releases['release-list']:
label_list = r['label-info-list']
for l in label_list:
try:
if l['label']['name'] != '[no label]':
labels.add(l['label']['id'])
except KeyError:
continue
labels = list(labels)
if len(labels) > 0:
return [{'value': l} for l in labels]
else:
logging.getLogger('root.features').warning(
f"I was not able to find any label for which artist {artist_musicbrainz_id['value']} has recorded"
)
return None
def get_artist_tracks_from_musicbrainz(cls, artist):
"""
Create Album, Track, and Solo records for artists we find in the MusicBrainzNGS API
:param artist: an artist's name as a string to search for
:return: Queryset of Solos
"""
search_results = mb.search_artists(artist)
best_result = search_results['artist-list'][0]
instrument = Solo.get_instrument_from_musicbrainz_tags(
best_result['tag-list'])
for album_dict in mb.browse_releases(best_result['id'],
includes=['recordings'
])['release-list']:
album = Album.objects.create(name=album_dict['title'],
artist=artist,
slug=slugify(album_dict['title']))
for track_dict in album_dict['medium-list'][0]['track-list']:
track = Track.objects.create(
album=album,
name=track_dict['recording']['title'],
track_number=track_dict['position'],
slug=slugify(track_dict['recording']['title']))
Solo.objects.create(track=track,
artist=artist,
instrument=instrument,
slug=slugify(artist))
return Solo.objects.filter(artist=artist)
def get_artist_tracks_from_musicbrainz(artist):
"""
Create Album, Track, and Solo records for artists we find in the MusicBrainzNGS API
:param artist: an artist's name as a string to search for
:return: Queryset of Solos
"""
search_results = mb.search_artists(artist)
best_result = search_results['artist-list'][0]
if 'jazz' not in [d['name'] for d in best_result['tag-list']]:
return Solo.objects.none()
instrument = Solo.get_instrument_from_musicbrainz_tags(best_result['tag-list'])
for album_dict in mb.browse_releases(best_result['id'], includes=['recordings'])['release-list']:
album = Album.objects.create(name=album_dict['title'], artist=artist, slug=slugify(album_dict['title']))
for track_dict in album_dict['medium-list'][0]['track-list']:
track = Track.objects.create(album=album, name=track_dict['recording']['title'],
track_number=track_dict['position'],
slug=slugify(track_dict['recording']['title']))
Solo.objects.create(track=track, artist=artist, instrument=instrument, slug=slugify(artist))
return Solo.objects.filter(artist=artist)
def browse_releases(artist_id=None,
release_group=None,
release_types=None,
limit=None,
offset=None,
includes=None):
"""Get all the releases by a certain artist and/or a release group.
You need to provide an artist's MusicBrainz ID or the Release Group's MusicBrainz ID
"""
if release_types is None:
release_types = []
key = cache.gen_key(artist_id, release_group, limit, offset,
*release_types, *includes)
releases = cache.get(key)
if not releases:
try:
api_resp = musicbrainzngs.browse_releases(
artist=artist_id,
release_type=release_types,
limit=limit,
offset=offset,
release_group=release_group,
includes=includes)
releases = api_resp.get('release-list')
except ResponseError as e:
if e.cause.code == 404:
return None
else:
raise InternalServerError(e.cause.msg)
cache.set(key=key, val=releases, time=DEFAULT_CACHE_EXPIRATION)
return releases
def test_browse_release(self):
artist = "47f67b22-affe-4fe1-9d25-853d69bc0ee3"
musicbrainzngs.browse_releases(artist=artist)
self.assertEqual("http://musicbrainz.org/ws/2/release/?artist=47f67b22-affe-4fe1-9d25-853d69bc0ee3", self.opener.get_url())
musicbrainzngs.browse_releases(track_artist=artist)
self.assertEqual("http://musicbrainz.org/ws/2/release/?track_artist=47f67b22-affe-4fe1-9d25-853d69bc0ee3", self.opener.get_url())
label = "713c4a95-6616-442b-9cf6-14e1ddfd5946"
musicbrainzngs.browse_releases(label=label)
self.assertEqual("http://musicbrainz.org/ws/2/release/?label=713c4a95-6616-442b-9cf6-14e1ddfd5946", self.opener.get_url())
recording = "7484fcfd-1968-4401-a44d-d1edcc580518"
musicbrainzngs.browse_releases(recording=recording)
self.assertEqual("http://musicbrainz.org/ws/2/release/?recording=7484fcfd-1968-4401-a44d-d1edcc580518", self.opener.get_url())
release_group = "1c1b54f7-e56a-3ce8-b62c-e45c378e7f76"
musicbrainzngs.browse_releases(release_group=release_group)
self.assertEqual("http://musicbrainz.org/ws/2/release/?release-group=1c1b54f7-e56a-3ce8-b62c-e45c378e7f76", self.opener.get_url())
def get_tracks_with_album_id(mb_id: str) -> List[Dict]:
try:
_metadata = mb.browse_releases(release_group=mb_id)["release-list"]
except Exception as e:
print(f"Unable to retrieve MB metadata for {mb_id}")
traceback.print_tb(e.__traceback__)
raise
else:
return _metadata
def find_all_tracks(self) -> None:
"""Find all tracks from all albums.
"""
self.all_albums = list()
total_albums = len(self.release_group_ids)
self.total_track_count = 0
with click.progressbar(
length=total_albums,
label=(
'Searching Musicbrainz for all tracks in all albums for '
f'{self.artist}'),
) as bar:
for _id, alb in self.release_group_ids.items():
resp_0 = addict.Dict(
musicbrainzngs.browse_releases(
release_group=_id,
release_type=['album'],
includes=['recordings'],
limit=100,
))
album_track_count = [
i['medium-list'][0]['track-count']
for i in resp_0['release-list']
]
self.total_track_count += max(album_track_count)
max_track_pos = album_track_count.index(max(album_track_count))
album_tracks = resp_0['release-list'][max_track_pos]
try:
album_year = resp_0['release-list'][
max_track_pos].date.split('-')[0]
except TypeError:
album_year = 'Missing'
album_tracks = addict.Dict((
alb + f' [{album_year}]',
[
i.recording.title for i in resp_0['release-list']
[max_track_pos]['medium-list'][0]['track-list']
],
))
self.all_albums.append(album_tracks)
bar.update(1)
# pprint(self.all_albums)
click.echo(f'Found {self.total_track_count} tracks across'
f' {len(self.release_group_ids)} albums for {self.artist}')
del resp_0
return self
def get_releases_for_release_group(release_group_id,
includes=RELEASE_INCLUDES):
with musicbrainz_lock:
logger.debug('Fetching release for rgid {}'.format(release_group_id))
search_results = musicbrainzngs.browse_releases(
release_group=release_group_id,
release_type='album',
includes=includes)['release-list']
return [album_info(release) for release in search_results]
def get_label_releases(label_id, offset, limit):
"""Get up to `limit` releases for label `label_id` starting from
offset `offset`."""
releases = musicbrainzngs.browse_releases(
label=label_id,
release_status=['official'],
release_type=['album', 'single', 'ep'],
includes=['release-groups'],
limit=limit,
offset=offset)
return releases
def handle(self, **options):
musicbrainzngs.set_useragent(settings.MB_APP, settings.MB_VERSION, settings.MB_CONTACT)
if options['changed']:
artists = Artist.objects.exclude(lastfm_mbid=F('mbid'))
else:
artists = Artist.objects\
.filter(lastfm_playcount__gt=settings.LASTFM_MIN_PLAY_COUNT, mbid__gt='')\
.order_by('lastfm_playcount')
for artist in artists:
print 'ARTIST: ', artist.title
if artist.updated and not (now() - artist.updated).days:
print 'updated recently'
continue
if options['changed']:
# deleting old releases and albums
artist.max_year = 0
for old_release in artist.releases.all():
old_release.delete()
for old_album in artist.albums.all():
old_album.delete()
artist.save()
# getting new releases
for release_type in settings.MB_RELEASE_TYPES:
limit = 100
offset = 0
parsed = 0
cont = True
while cont:
releases = musicbrainzngs.browse_releases(artist=artist.mbid,
release_type=[release_type],
release_status=settings.MB_RELEASE_STATUSES,
limit=limit,
offset=offset)
for release in releases['release-list']:
parsed += 1
album, created = Release.objects.get_or_create(mbid=release.get('id'), artist=artist)
if created:
print 'NEW: ', release.get('id'), release.get('title')
album.date = release.get('date')
if album.date:
album.year = release.get('date').split('-')[0]
album.title = release.get('title')
album.country = release.get('country')
album.type = release_type
album.save()
if parsed < releases['release-count']:
offset += limit
else:
cont = False
artist.updated = now()
artist.save()
print 'FINISHED'
def get_artist_releases(artist_mbid):
limit = 100
offset = 0
releases = []
release_groups = []
page = 1
result = mbz.browse_releases(
artist=artist_mbid,
limit=limit,
offset=offset,
includes=["release-groups", "artist-credits"],
release_status=["official"],
)
releases += result["release-list"]
if "release-count" in result:
count = result["release-count"]
while len(releases) < count:
offset += limit
page += 1
result = mbz.browse_releases(
artist=artist_mbid,
limit=limit,
offset=offset,
includes=["release-groups", "artist-credits"],
release_status=["official"],
)
releases += result["release-list"]
for release in releases:
if release not in release_groups:
release_groups.append(release["release-group"])
return {"status": 200, "releases": release_groups}
def MBID_releases_iterator(mbid):
"""
Returns a dictionary with all releases in the form {title:mbid} for a given artist MBID
"""
releases = {}
rel = m.browse_releases(mbid)['release-list']
# print rel
no_rel = len(rel)
for i in xrange(no_rel):
title = rel[i]['title']
mbid_title = rel[i]['id']
releases[title] = mbid_title
# print releases
return releases
def get_all_artist_songs(artist):
'''Get a list of songs for a given artist'''
artist_id = get_artist_id(artist)
offset = 0
songs = []
result_len = BROWSE_LIMIT
while result_len == BROWSE_LIMIT:
result = mb.browse_releases(artist=artist_id,
release_type=['single'],
limit=BROWSE_LIMIT,
offset=offset)['release-list']
result_len = len(result)
offset += result_len
songs.extend([s['title'] for s in result])
return list(set(songs))
def getReleaseCount(self, release_group_id):
"""
Helper function used to get the count of releases for a release-group by it's ID
:param release_group_id: This is the id of the release-group
:return release_count: The number of releases for that release-group
"""
try:
release_group_releases = musicbrainzngs.browse_releases(
release_group=release_group_id)
count = release_group_releases['release-count']
except:
raise serializers.ValidationError({
"Too much music":
"Seems like everybody wants to know how many albums blink 182 released, \
At least they are not looking for Britney. Give us a second a try again"
})
return count
def getArtistAlbums(self, artist_mbid): if artist_mbid in self.artist_cache: return self.artist_cache[artist_mbid] data = musicbrainzngs.browse_releases(artist=artist_mbid, includes=['release-groups', 'recordings']) ret = [] for rel in data['release-list']: if rel['status'] != 'Official': continue if rel['release-group']['type'] != 'Album': continue format = rel['medium-list'][0]['format'] if not((format == 'CD') | (format == 'Digital')): continue ret.append(rel) self.artist_cache[artist_mbid] = ret return ret
def import_releases(artist, mb_id):
releases = mb.browse_releases(artist=mb_id, release_type='album')
for release in releases['release-list']:
title = release['title']
try:
date = convert_time(release['date'])
except:
date = None
try:
mb_id = release['id']
except:
mb_id = '0'
result = models.Album.objects.filter(musicbrainz_id=mb_id)
if len(result) > 0:
return
album = models.Album(artist=artist, name=title,
musicbrainz_id=mb_id)
album.save()
def artist_self_releasing_records(
artist_musicbrainz_id) -> 'artist_self_releasing_records':
"""Tell if the artist have self released records (i.e. without label) or not
Arguments:
artist_musicbrainz_id {str} --
Returns:
bool --
"""
releases = mz.browse_releases(artist=artist_musicbrainz_id['value'],
includes=['labels'])
for r in releases['release-list']:
label_list = r['label-info-list']
for l in label_list:
try:
if l['label']['name'] == '[no label]':
return {'value': True}
except KeyError:
continue
def first_release_date(self, group_id):
"""Return the first release date in the release group, or None."""
dates = []
limit = 25
offset = 0
while True:
try:
response = mb_client.browse_releases(release_group=group_id,
limit=limit,
offset=offset)
releases = response['release-list']
dates.extend([r['date'] for r in releases if 'date' in r])
if len(releases) < limit:
break
else:
offset += limit
except mb_client.ResponseError as e:
raise MusicBrainzError from e
if dates:
return min(dates)
else:
return None
def get_releases(cls, mbid, rtypes=None):
"""
@param: Artist mbid
"""
limit = 100
rtypes = ['album'] if not rtypes else rtypes
result = musicbrainzngs.browse_releases(artist=mbid,
release_type=rtypes,
limit=limit)
page_releases = result['release-list']
albums = []
known_titles = []
for release in page_releases:
if release.get('status', '') == 'Official':
mbid = release.get('id', '')
date = release.get('date', '')
date = date.split('-')[0]
title = release.get('title', '')
if title and not title in known_titles:
known_titles.append(title)
albums.append({'title': title, 'mbid': mbid, 'date': date})
return albums
def music_get_artist_albums(artist_id: str, locale: str):
releases = musicbrainzngs.browse_releases(artist_id,
limit=100)["release-list"]
artist = musicbrainzngs.get_artist_by_id(artist_id)["artist"]
artist_name: str = artist["name"]
doc: Document = minidom.Document()
feed: Element = create_feed(doc, artist_id, artist_name, request.endpoint)
for release in releases:
id: str = release["id"]
title: str = release["title"]
entry: Element = create_entry(
doc, title, id,
f"/v3.2/{locale}/music/artist/{artist_id}/albums/{id}")
feed.appendChild(entry)
# Add front cover
image_elem: Element = doc.createElement("image")
image_id_elem: Element = create_id(doc, id)
image_elem.appendChild(image_id_elem)
entry.appendChild(image_elem)
# Create primaryArtist element
primary_artist_elem: Element = doc.createElement("primaryArtist")
artist_id_element: Element = doc.createElement("id")
set_element_value(artist_id_element, artist_id)
primary_artist_elem.appendChild(artist_id_element)
artist_name_element: Element = doc.createElement("name")
set_element_value(artist_name_element, artist_name)
primary_artist_elem.appendChild(artist_name_element)
entry.appendChild(primary_artist_elem)
#doc.appendChild(feed)
xml_str = doc.toprettyxml(indent="\t")
return Response(xml_str, mimetype=MIME_XML)
def get_release_list(artist_str):
""" Gets a release list from musicbrains API. """
username = '******'
password = '******'
musicbrainzngs.set_useragent(username, password)
artist_list = musicbrainzngs.search_artists(artist=artist_str)['artist-list']
artist = sorted(artist_list, reverse=True, key=lambda artist:int(artist['ext:score']))[0]
artist_id = artist['id']
limit = 100
offset = 0
release_list = []
release_count = 1
while offset < release_count:
print 'Requesting tracks {0} - {1}'.format(str(offset), str(offset+limit))
result = musicbrainzngs.browse_releases(artist=artist_id, release_status=['official'], release_type=['album'], includes=['recordings'], limit=limit, offset=offset)
release_count = result['release-count']
release_list += result['release-list']
offset += limit
return release_list
def _artist_credit_fallback_alias(artist_credit, release={}):
"""Given an artist credit block, attempt to find an alias for the artist with the user's preferred locale
from all the artist-credits for the artist.
Optionally, we may have data for a release already, so we can try it first.
Returns an alias for the artist, or the original artist, if not found.
"""
artist = artist_credit["artist"]
artist_name = artist['name']
credit_name = artist_credit.get('name')
release_lang = release.get('text-representation', {}).get('language')
# Comparing by ISO639-3, which musicbrainz seems to use for release languages (but not for aliases!)
release_lang = ISO_639_1_TO_3.get(release_lang, release_lang)
preferred_langs = [
ISO_639_1_TO_3.get(x, x) for x in config['import']['languages']
]
# If the associated release matches the primary config locale, and has an additional credit name, use it.
if release_lang and (release_lang == preferred_langs[0]) and credit_name:
return credit_name
# Use the canonical artist name as fallback
artist_name = artist['name']
# Lookup additional data on how the artist is credited on their releases
# This could perhaps be cached somehow, if repeated api calls are a problem...
# (Even just a cache of the last artist would make a difference in this case)
try:
res = musicbrainzngs.browse_releases(artist=artist['id'],
includes='artist-credits')
except musicbrainzngs.ResponseError:
return artist_name
releases = res['release-list']
# Look through artist releases to find an artist-credits matching the users
# preferred languages
name_candidates = {}
for release in releases:
release_lang = release.get('text-representation', {}).get('language')
release_lang = ISO_639_1_TO_3.get(release_lang, release_lang)
# Skip releases without a language
if release_lang is None:
continue
if release_lang in preferred_langs:
# Get artist-credit name matching artist-id
credit_name = None
for credit in release['artist-credit']:
# Skip over credits like x, ft., and, etc
if (isinstance(credit, six.string_types)):
continue
if credit['artist']['id'] == artist['id']:
# Differing credit name is under credit['name'] otherwise, will just give the usual artist
# details, without this additional tag
credit_name = credit.get('name')
break
if credit_name is None:
continue
name_candidates.setdefault(release_lang, []).append(credit_name)
# Choose from found credit-names based on preferred languages
for lang in preferred_langs:
names = name_candidates.get(lang)
if names:
# Just return the first name matching a locale
# alternatively, we could perhaps pick one based on the newest release, for example
return names[0]
# Nothing valid found, fallback to what we had already
return artist_name
# print(artist["country"])
# print(artist["begin-area"])
# print(artist["type"])
# print(artist["sort-name"])
label = "71247f6b-fd24-4a56-89a2-23512f006f0c"
limit = 100
offset = 0
releases = []
page = 1
print("fetching page number %d.." % page)
result = musicbrainzngs.browse_releases(label=label, includes=["labels"],
release_type=["album"], limit=limit)
page_releases = result['release-list']
print(page_releases)
releases += page_releases
# release-count is only available starting with musicbrainzngs 0.5
if "release-count" in result:
count = result['release-count']
print("")
while len(page_releases) >= limit:
offset += limit
page += 1
print("fetching page number %d.." % page)
# get genres for artist (not more than 3)
try:
# sort tags by count
newlist = sorted(artist['tag-list'],
key=lambda k: int(k['count']),
reverse=True)
genre_list = [k['name'] for k in newlist]
genre = '#'.join(
genre_list if len(genre_list) < 4 else genre_list[:3])
except KeyError:
print("SKIIIIIP : ARTIST HAS NO GENRE TAGS")
continue
# get albums
# length of all releases from this artist
album_query = brainz.browse_releases(artist=artist["id"])
album_query_len = album_query['release-count']
# initialize temp variables
saved_releases = []
release_offset = release_start // release_limitsize * release_limitsize
# same as above, brainz.search has limit of 100
while release_offset < album_query_len:
for release_index, release in enumerate(
brainz.browse_releases(
artist=artist['id'],
offset=release_offset + release_start,
limit=release_limitsize)['release-list']):
# calculate current position for progress debug
current_artist = artist_offset + artist_index + start_artist % artist_limitsize
current_release = release_offset + release_index + release_start % release_limitsize
print(
def getAlbumArtistNames(album,artist, apihandle, song=None):
'''
Given a supposed album and artist, determine the real ones
'''
def searchWhatAlbums(args):
if len(args)==0:
return []
whatResponse = apihandle.request(action='browse',searchstr=args[0])
if whatResponse['status']=='success':
args.pop(0)
return searchWhatAlbums(args)+[x for x in whatResponse['response']['results'] if 'artist' in x and 'groupName' in x]
return []
mbAlbums = []
parens = re.compile('[\(\[].*[\)\]]')
if song is not None:
includes = ['recordings']
artists = set(re.split(' &|and|ft\.?|featuring|feat\.? ',artist))
if len(artists) > 1:
artists.add(artist)
for ar in artists:
mbAlbums += [dict(list(x.items())+[("artist-credit-phrase", releases['artist-credit-phrase'])]) for releases in mb.search_recordings(query=song, limit=int(50*mbMultiplier), artistname=mbquote(ar), release=mbquote(album))['recording-list'] for x in releases['release-list']]
mbAlbums += mb.search_releases(artist=mbquote(ar),query=mbquote(album),limit=int(25*mbMultiplier))['release-list']
lastfmres = [x['mbid'] for x in lookup('lastfm','songsearch',{'artist':ar, 'song':song})['results']['trackmatches']['track'] if 'mbid' in x and len(x['mbid'])>0]
if len(lastfmres)>0:
for lastfmRecId in set(lastfmres):
try:
lastfmAlbum = mb.get_recording_by_id(id=lastfmRecId,includes=['releases','artist-credits'])
for alb in lastfmAlbum['recording'].pop('release-list'):
alb['medium-list'] = [{}]
alb['medium-list'][0]['track-list'] = []
alb['medium-list'][0]['track-list'].append(lastfmAlbum)
alb['artist-credit-phrase'] = lastfmAlbum['recording']['artist-credit-phrase']
mbAlbums.append(alb)
except Exception as e:
print(e)
else:
includes = []
mbArtists = mb.search_artists(query=mbquote(artist),limit=int(40*mbMultiplier))['artist-list']
mbAlbums += mb.search_releases(artist=mbquote(artist),query=mbquote(album),limit=int(50*mbMultiplier))['release-list']
for mbArtist in mbArtists:
if Levenshtein.ratio(artist.lower(),mbArtist['name'].lower()) > 0.75:
mbAlbums+=[ dict(list(x.items())+[('artist-credit-phrase',mbArtist['name'])]) for x in mb.browse_releases(artist=mbArtist['id'],includes=includes,limit=25)['release-list']]
if (len(album)<7 and ('/' in album or ' & ' in album) and 's' in album.lower() and 't' in album.lower()) or ('self' in album.lower() and 'titled' in album.lower()):
mbAlbums += mb.search_releases(artist=mbquote(artist),query=mbquote(artist),limit=int(25*mbMultiplier))['release-list']
temp = []
for x in mbAlbums[:]:
if x["id"] in temp and not ('medium-list' in x and len(x['medium-list'])>0 and all('track-list' in z and len(z['track-list'])>0 for z in x['medium-list'])):
mbAlbums.remove(x)
else:
temp.append(x['id'])
print("Done searching musicbrainz for album suggestions, have "+str(len(mbAlbums))+" to rank")
ranks = {}
for x in mbAlbums:
ranks[x['id']] = Levenshtein.ratio(album.lower(),x['title'].lower())
if song is not None:
x['song'] = {}
temp = ([(y['recording']['title'] if 'recording' in y else y['title'],
int(float(
y['recording']['length'] if 'recording' in y and 'length' in y['recording']
else (y['track_or_recording_length'] if 'track_or_recording_length' in x
else y['length'] if 'length' in x else 0)
)/1000.))
for tracklist in x['medium-list']
for y in tracklist['track-list']]
if 'medium-list' in x and len(x['medium-list'])>0 and all('track-list' in z and len(z['track-list'])>0 for z in x['medium-list'])
else getSongs(
{"artist":x['artist-credit-phrase'],
"groupName":x['title']}))
x['song']['name'], x['song']['duration'] = (max(temp,
key=lambda y: Levenshtein.ratio(y[0].lower(),song.lower())) if len(temp)>0 else ("",-1))
if ranks[x['id']] < Levenshtein.ratio(x['song']['name'].lower(),song.lower()):
ranks[x['id']] /= 6
ranks[x['id']] += (Levenshtein.ratio(x['song']['name'].lower(),song.lower())
+ Levenshtein.ratio(parens.sub('',x['song']['name'].lower()),parens.sub('',song.lower())) )*5/12
else:
ranks[x['id']] /= 3
ranks[x['id']] += (Levenshtein.ratio(x['song']['name'].lower(),song.lower())
+ Levenshtein.ratio(parens.sub('',x['song']['name'].lower()),parens.sub('',song.lower())) )/3
ranks[x['id']] += Levenshtein.ratio(artist.lower(),x['artist-credit-phrase'].lower())*7/6
if len(ranks) == 0:
return None
mbAlbumId, mbAlbumRank = max(ranks.items(),key=(lambda x:x[1]))
mbAlbum = [x for x in mbAlbums if x['id']==mbAlbumId][0]
print("For the artist and album derived from the provided dir ("+artist+" and "+album+" respectively),\nthe following artist and album was matched on musicbrains:")
print("Artist: "+mbAlbum['artist-credit-phrase'])
print("Album: "+mbAlbum['title'])
whatAlbums = searchWhatAlbums([mbAlbum['title']+' '+mbAlbum['artist-credit-phrase'], artist+' '+album])
if len(whatAlbums) == 0:
whatAlbums = searchWhatAlbums([artist,album,mbAlbum['title'],mbAlbum['artist-credit-phrase']])
if len(whatAlbums) == 0:
return None
whatAlbums = sorted(whatAlbums, key=(lambda x:
Levenshtein.ratio(x['groupName'],mbAlbum['title'])
+Levenshtein.ratio(x['groupName'].lower(),album.lower())*3/8
+Levenshtein.ratio(x['artist'],mbAlbum['artist-credit-phrase'])
+Levenshtein.ratio(x['artist'].lower(),artist.lower())*5/8),
reverse=True)#[:min(10,len(whatAlbums))]
whatAlbum = whatAlbums[0]
whatAlbum['artist-credit-phrase'] = mbAlbum['artist-credit-phrase']
if song is not None:
whatAlbum['song'] = mbAlbum['song']
print("For the album and artist found on musicbrainz, the following torrent group was found on what:")
print("Artist: "+whatAlbum['artist'])
print("Album: "+whatAlbum['groupName'])
return whatAlbum
import musicbrainzngs
import requests
def sources(recordingid):
res = requests.get('http://185.97.32.250:8468/mbid:{}'.format(recordingid))
#import pdb;pdb.set_trace()
if not b'None' in res.content:
return res.content.decode('utf-8')
musicbrainzngs.set_useragent("mb.py", "0", contact="*****@*****.**")
res = musicbrainzngs.search_artists("Ablaze")
for artist in res['artist-list']:
print(artist['name'], artist['id'])
#import pdb;pdb.set_trace()
res = musicbrainzngs.browse_releases(artist='d2c0d69e-e3ca-45a4-a540-6ce42c617599', limit=100)
for release in res['release-list']:
print(release['title'])
recordings = musicbrainzngs.browse_recordings(release=release['id'])
for rec in recordings['recording-list']:
# import pdb;pdb.set_trace()
print("\t\t",rec['title'],'\t', end='')
print(sources(rec['id']))
#import pdb;pdb.set_trace()
def getMusicbrainzArtistMetadata(artist):
artist_md.musicbrainz_id = artist['id']
if 'country' in artist:
if artist['country'] == 'US':
artist_md.is_american = True
artist_md.is_german = False
artist_md.is_other_country = False
elif artist['country'] == 'DE':
artist_md.is_german = True
artist_md.is_american = False
artist_md.is_other_country = False
else:
artist_md.is_other_country = True
artist_md.is_american = False
artist_md.is_german = False
artist_md.country = artist['country']
else:
pass # TODO: get country
if 'area' in artist and 'iso-3166-1-code-list' in artist['area']:
artist_md.area = artist['area']['iso-3166-1-code-list']
else:
pass # TODO: get country
if 'type' in artist:
if artist['type'] != "Person":
artist_md.is_group = True
artist_md.is_male = False
artist_md.is_female = False
else:
artist_md.is_group = False
if 'gender' in artist:
if artist['gender'] == 'Male':
artist_md.is_male = True
artist_md.is_female = False
artist_md.is_group = False
elif artist['gender'] == 'Female':
artist_md.is_male = False
artist_md.is_female = True
artist_md.is_group = False
if 'life-span' in artist:
if 'begin' in artist['life-span']:
if 'end' in artist['life-span']:
artist_md.life_span = utils.getActivity(start=artist['life-span']['begin'],
end=artist['life-span']['end'])
else:
try:
artist_md.life_span = utils.getActivity(start=artist['life-span']['begin'])
except ValueError:
print colored("| Date error...", 'red')
artist_md.error = True
'''
musicbrainz uses a rating based on 5 (0 = bad, 5= good) but I want a float between 0 and 100
'''
if 'rating' in artist:
artist_md.buffer['popularity'].append(float(artist['rating']['rating']) * 20)
artist_md.buffer['recording_count'].append(artist['recording-count'])
artist_md.buffer['release_count'].append(artist['release-count'])
artist_md.buffer['work_count'].append(artist['work-count'])
if 'tag-list' in artist:
for tag in artist['tag-list']:
artist_md.addTag(tag['name']) # TODO: 'count' instead of weight
releases = musicbrainzngs.browse_releases(artist['id'], includes=['labels'], limit=50)
for release in releases['release-list']:
if 'text-representation' in release and 'language' in release['text-representation']:
artist_md.addLanguage(release['text-representation']['language'])
if 'label-info-list' in release:
for label in release['label-info-list']:
if 'label' in label:
artist_md.addLabel({'name': label['label']['name'], 'release_count': None}, parent_label=None,
sublabels=None) # TODO: get missing data
def getMusicbrainzArtistMetadata(artist):
artist_md.musicbrainz_id = artist['id']
if 'country' in artist:
if artist['country'] == 'US':
artist_md.is_american = True
artist_md.is_german = False
artist_md.is_other_country = False
elif artist['country'] == 'DE':
artist_md.is_german = True
artist_md.is_american = False
artist_md.is_other_country = False
else:
artist_md.is_other_country = True
artist_md.is_american = False
artist_md.is_german = False
artist_md.country = artist['country']
else:
pass # TODO: get country
if 'area' in artist and 'iso-3166-1-code-list' in artist['area']:
artist_md.area = artist['area']['iso-3166-1-code-list']
else:
pass # TODO: get country
if 'type' in artist:
if artist['type'] != "Person":
artist_md.is_group = True
artist_md.is_male = False
artist_md.is_female = False
else:
artist_md.is_group = False
if 'gender' in artist:
if artist['gender'] == 'Male':
artist_md.is_male = True
artist_md.is_female = False
artist_md.is_group = False
elif artist['gender'] == 'Female':
artist_md.is_male = False
artist_md.is_female = True
artist_md.is_group = False
if 'life-span' in artist:
if 'begin' in artist['life-span']:
if 'end' in artist['life-span']:
artist_md.life_span = utils.getActivity(
start=artist['life-span']['begin'],
end=artist['life-span']['end'])
else:
try:
artist_md.life_span = utils.getActivity(
start=artist['life-span']['begin'])
except ValueError:
print colored("| Date error...", 'red')
artist_md.error = True
'''
musicbrainz uses a rating based on 5 (0 = bad, 5= good) but I want a float between 0 and 100
'''
if 'rating' in artist:
artist_md.buffer['popularity'].append(
float(artist['rating']['rating']) * 20)
artist_md.buffer['recording_count'].append(artist['recording-count'])
artist_md.buffer['release_count'].append(artist['release-count'])
artist_md.buffer['work_count'].append(artist['work-count'])
if 'tag-list' in artist:
for tag in artist['tag-list']:
artist_md.addTag(tag['name']) # TODO: 'count' instead of weight
releases = musicbrainzngs.browse_releases(artist['id'],
includes=['labels'],
limit=50)
for release in releases['release-list']:
if 'text-representation' in release and 'language' in release[
'text-representation']:
artist_md.addLanguage(release['text-representation']['language'])
if 'label-info-list' in release:
for label in release['label-info-list']:
if 'label' in label:
artist_md.addLabel(
{
'name': label['label']['name'],
'release_count': None
},
parent_label=None,
sublabels=None) # TODO: get missing data
def count_release(self, release_group_id):
result = musicbrainzngs.browse_releases(release_group=release_group_id)
return result['release-count']
def _missing_albums(self, lib, query):
"""Print a listing of albums missing from each artist in the library
matching query.
"""
total = self.config['total'].get()
fmt = config['format_album'].get()
release_status = self.config['release_status'].get()
release_type = self.config['release_type'].get()
albums = lib.albums(query)
# build dict mapping artist to list of their albums in library
albums_by_artist = defaultdict(list)
for alb in albums:
artist = (alb['albumartist'], alb['mb_albumartistid'])
albums_by_artist[artist].append(alb)
total_missing = 0
# build dict mapping artist to list of all albums
for artist, albums in albums_by_artist.items():
if artist[1] is None or artist[1] == "":
albs_no_mbid = [u"'" + a['album'] + u"'" for a in albums]
self._log.info(
u"No musicbrainz ID for artist '{}' found in album(s) {}; "
"skipping", artist[0], u", ".join(albs_no_mbid)
)
continue
try:
resp = musicbrainzngs.browse_releases(
artist=artist[1],
release_status=release_status,
release_type=release_type,
includes=[
"artist-credits",
"labels",
"recordings",
"isrcs",
"release-groups",
"media",
"discids",
"area-rels",
"artist-rels",
"label-rels",
"place-rels",
"event-rels",
"recording-rels",
"release-rels",
"release-group-rels",
"series-rels",
"url-rels",
"work-rels",
"instrument-rels"
]
)
release_groups = resp['release-list']
except MusicBrainzError as err:
self._log.info(
u"Couldn't fetch info for artist '{}' ({}) - '{}'",
artist[0], artist[1], err
)
continue
missing = []
present = []
for rg in release_groups:
missing.append(rg)
for alb in albums:
if alb['mb_releasegroupid'] == rg.get("release-group", {}).get("id"):
missing.remove(rg)
present.append(rg)
break
total_missing += len(missing)
if total:
continue
for r in missing:
#self._log.info(u"{}\n\n", r)
item = _album_item(r)
print_(format(item, fmt))
if total:
print(total_missing)
def populate_db(mbdb, options, irank_db):
if options.threshold:
artists = irank_db.execute('select artist from songs where artist <> "" group by artist having count(path) > ? order by count(path)', [options.threshold])
else:
artists = irank_db.execute('select artist from songs where artist <> "" group by artist')
if options.target:
assert options.max_age, "--max-age must be provided when using --target"
min_date = TODAY - datetime.timedelta(days=options.max_age)
logging.debug("min_date: %r" % (min_date,))
# we're not doing all artists, just the `n` oldest
artists = list(artists)
logging.debug("total artists: %d", len(artists))
def age(artist):
checked = list(mbdb.execute('select checked from artists where name = ?', [artist]))
if not checked: return EPOCH
date = checked[0][0]
if date == 'now':
# oops...
return to_date('2015-02-01')
return to_date(date)
artists_with_age = [(artist, age(artist[0])) for artist in artists]
artists_with_age = sorted(artists_with_age, key=lambda pair:pair[1])
if options.min_age:
max_date = TODAY - datetime.timedelta(days=options.min_age)
logging.debug("max_date: %r", max_date)
artists_with_age = list(itertools.takewhile(lambda pair: pair[1] < max_date, artists_with_age))
logging.debug(
"there are %d artists which were checked more than %d days ago" %
(len(artists_with_age), options.min_age))
selection = list(itertools.takewhile(lambda pair: pair[1] < min_date, artists_with_age))
logging.debug(
"there are %d artists which were checked more than %d days ago" %
(len(selection), options.max_age))
if len(selection) > options.target:
logging.warn(
"processing %d artists which were checked more than %d days ago" %
(len(selection), options.max_age))
else:
selection = artists_with_age[:options.target]
artists = [pair[0] for pair in selection]
# logging.debug('%r', selection)
logging.debug("Processing %d artists", len(artists))
now = format_date(TODAY)
for (artist,) in artists:
logging.debug("getting ID for artist %s", artist)
results = mb.search_artists(artist=artist, strict=False, limit=10)['artist-list']
if len(results) == 0:
err("WARN: no such artist: %s" % artist)
continue
if len(results) > 1:
logging.info("multiple artists: %s", ' / '.join(list(map(lambda artist: artist['name'], results))))
result = results[0]
artist_id = result['id']
mbdb.execute('insert or replace into artists (name, id, checked) values (?,?,?)', (artist, artist_id, now))
# OK, id saved to the DB now.
offset=0
limit=100
releases = {}
while True:
logging.info("getting releases[%d:%d] for artist %s [%s]", offset, offset+limit, artist, artist_id)
recordings = mb.browse_releases(artist=artist_id, release_type=['album', 'ep'], includes=['url-rels'], offset=offset, limit=limit)['release-list']
# recordings = sorted(recordings, key=lambda rec: rec.get('date', ''))
logging.info("got %d recordings", len(recordings))
for title, recordings in itertools.groupby(recordings, lambda x: x['title']):
recordings = list(recordings)
dates = list(map(lambda rec: rec.get('date', ''), recordings))
# err(repr((title, dates)))
longest_timestamp = len(max(dates))
if not longest_timestamp:
continue
earliest_date = min(filter(lambda date: len(date) == longest_timestamp, dates))
if title in releases and releases[title] <= earliest_date:
continue
releases[title] = earliest_date
if len(recordings) < limit: break # we got less than `limit`, so there aren't any more
offset += len(recordings)
for title, date in sorted(releases.items(), key=lambda pair:pair[1]):
date = format_date(to_date(date))
mbdb.execute('insert or replace into releases (artist_id, title, artist, date) VALUES (?,?,?,?)', (artist_id, title, artist, date))
# err("%s - %s (released %s)" % (artist, title, date))
mbdb.execute('update artists set checked=? where name=? and id=?', (now, artist, artist_id))
mbdb.commit()
def get_new_releases(rgid, includeExtras=False, forcefull=False):
myDB = db.DBConnection()
results = []
release_status = "official"
if includeExtras and not headphones.CONFIG.OFFICIAL_RELEASES_ONLY:
release_status = []
try:
limit = 100
newResults = None
while newResults is None or len(newResults) >= limit:
with mb_lock:
newResults = musicbrainzngs.browse_releases(
release_group=rgid,
includes=["artist-credits", "labels", "recordings", "release-groups", "media"],
release_status=release_status,
limit=limit,
offset=len(results),
)
if "release-list" not in newResults:
break # may want to raise an exception here instead ?
newResults = newResults["release-list"]
results += newResults
except musicbrainzngs.WebServiceError as e:
logger.warn(
'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))
)
mb_lock.snooze(5)
return False
if not results or len(results) == 0:
return False
# Clean all references to releases in dB that are no longer referenced in musicbrainz
release_list = []
force_repackage1 = 0
if len(results) != 0:
for release_mark in results:
release_list.append(unicode(release_mark["id"]))
release_title = release_mark["title"]
remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid])
if remove_missing_releases:
for items in remove_missing_releases:
if items["ReleaseID"] not in release_list and items["ReleaseID"] != rgid:
# Remove all from albums/tracks that aren't in release
myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items["ReleaseID"]])
myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items["ReleaseID"]])
myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items["ReleaseID"]])
myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items["ReleaseID"]])
logger.info("Removing all references to release %s to reflect MusicBrainz" % items["ReleaseID"])
force_repackage1 = 1
else:
logger.info(
"There was either an error pulling data from MusicBrainz or there might not be any releases for this category"
)
num_new_releases = 0
for releasedata in results:
release = {}
rel_id_check = releasedata["id"]
album_checker = myDB.action("SELECT * from allalbums WHERE ReleaseID=?", [rel_id_check]).fetchone()
if not album_checker or forcefull:
# DELETE all references to this release since we're updating it anyway.
myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rel_id_check])
myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rel_id_check])
release["AlbumTitle"] = unicode(releasedata["title"])
release["AlbumID"] = unicode(rgid)
release["AlbumASIN"] = unicode(releasedata["asin"]) if "asin" in releasedata else None
release["ReleaseDate"] = unicode(releasedata["date"]) if "date" in releasedata else None
release["ReleaseID"] = releasedata["id"]
if "release-group" not in releasedata:
raise Exception("No release group associated with release id " + releasedata["id"] + " album id" + rgid)
release["Type"] = unicode(releasedata["release-group"]["type"])
if release["Type"] == "Album" and "secondary-type-list" in releasedata["release-group"]:
secondary_type = unicode(releasedata["release-group"]["secondary-type-list"][0])
if secondary_type != release["Type"]:
release["Type"] = secondary_type
# making the assumption that the most important artist will be first in the list
if "artist-credit" in releasedata:
release["ArtistID"] = unicode(releasedata["artist-credit"][0]["artist"]["id"])
release["ArtistName"] = unicode(releasedata["artist-credit-phrase"])
else:
logger.warn("Release " + releasedata["id"] + " has no Artists associated.")
return False
release["ReleaseCountry"] = unicode(releasedata["country"]) if "country" in releasedata else u"Unknown"
# assuming that the list will contain media and that the format will be consistent
try:
additional_medium = ""
for position in releasedata["medium-list"]:
if position["format"] == releasedata["medium-list"][0]["format"]:
medium_count = int(position["position"])
else:
additional_medium = additional_medium + " + " + position["format"]
if medium_count == 1:
disc_number = ""
else:
disc_number = str(medium_count) + "x"
packaged_medium = disc_number + releasedata["medium-list"][0]["format"] + additional_medium
release["ReleaseFormat"] = unicode(packaged_medium)
except:
release["ReleaseFormat"] = u"Unknown"
release["Tracks"] = getTracksFromRelease(releasedata)
# What we're doing here now is first updating the allalbums & alltracks table to the most
# current info, then moving the appropriate release into the album table and its associated
# tracks into the tracks table
controlValueDict = {"ReleaseID": release["ReleaseID"]}
newValueDict = {
"ArtistID": release["ArtistID"],
"ArtistName": release["ArtistName"],
"AlbumTitle": release["AlbumTitle"],
"AlbumID": release["AlbumID"],
"AlbumASIN": release["AlbumASIN"],
"ReleaseDate": release["ReleaseDate"],
"Type": release["Type"],
"ReleaseCountry": release["ReleaseCountry"],
"ReleaseFormat": release["ReleaseFormat"],
}
myDB.upsert("allalbums", newValueDict, controlValueDict)
for track in release["Tracks"]:
cleanname = helpers.clean_name(
release["ArtistName"] + " " + release["AlbumTitle"] + " " + track["title"]
)
controlValueDict = {"TrackID": track["id"], "ReleaseID": release["ReleaseID"]}
newValueDict = {
"ArtistID": release["ArtistID"],
"ArtistName": release["ArtistName"],
"AlbumTitle": release["AlbumTitle"],
"AlbumID": release["AlbumID"],
"AlbumASIN": release["AlbumASIN"],
"TrackTitle": track["title"],
"TrackDuration": track["duration"],
"TrackNumber": track["number"],
"CleanName": cleanname,
}
match = myDB.action(
"SELECT Location, BitRate, Format from have WHERE CleanName=?", [cleanname]
).fetchone()
if not match:
match = myDB.action(
"SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?",
[release["ArtistName"], release["AlbumTitle"], track["title"]],
).fetchone()
# if not match:
# match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
if match:
newValueDict["Location"] = match["Location"]
newValueDict["BitRate"] = match["BitRate"]
newValueDict["Format"] = match["Format"]
# myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
myDB.action("UPDATE have SET Matched=? WHERE Location=?", (release["AlbumID"], match["Location"]))
myDB.upsert("alltracks", newValueDict, controlValueDict)
num_new_releases = num_new_releases + 1
if album_checker:
logger.info(
"[%s] Existing release %s (%s) updated"
% (release["ArtistName"], release["AlbumTitle"], rel_id_check)
)
else:
logger.info(
"[%s] New release %s (%s) added" % (release["ArtistName"], release["AlbumTitle"], rel_id_check)
)
if force_repackage1 == 1:
num_new_releases = -1
logger.info(
"[%s] Forcing repackage of %s, since dB releases have been removed"
% (release["ArtistName"], release_title)
)
else:
num_new_releases = num_new_releases
return num_new_releases
def get_artist_info(mbid):
result = musicbrainzngs.get_artist_by_id(mbid, ['url-rels', 'tags'])
releases = musicbrainzngs.browse_releases(artist=mbid, limit=100)
result['artist']['release-list'] = releases['release-list']
return result
async def _releases(artist_id, offset=0):
return mbz.browse_releases(artist=artist_id,
release_type=['album', 'single'],
includes='recordings',
offset=offset,
limit=100)
def get_new_releases(rgid,includeExtras=False,forcefull=False):
myDB = db.DBConnection()
results = []
try:
limit = 100
newResults = None
while newResults == None or len(newResults) >= limit:
newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results))
if 'release-list' not in newResults:
break #may want to raise an exception here instead ?
newResults = newResults['release-list']
results += newResults
except musicbrainzngs.WebServiceError as e:
logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e)))
time.sleep(5)
return False
if not results or len(results) == 0:
return False
#Clean all references to releases in dB that are no longer referenced in musicbrainz
release_list = []
force_repackage1 = 0
if len(results) != 0:
for release_mark in results:
release_list.append(unicode(release_mark['id']))
release_title = release_mark['title']
remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid])
if remove_missing_releases:
for items in remove_missing_releases:
if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid:
# Remove all from albums/tracks that aren't in release
myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']])
myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']])
myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']])
myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']])
logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID'])
force_repackage1 = 1
else:
logger.info("There was either an error pulling data from MusicBrainz or there might not be any releases for this category")
num_new_releases = 0
for releasedata in results:
#releasedata.get will return None if it doesn't have a status
#all official releases should have the Official status included
if not includeExtras and releasedata.get('status') != 'Official':
continue
release = {}
rel_id_check = releasedata['id']
artistid = unicode(releasedata['artist-credit'][0]['artist']['id'])
album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone()
if not album_checker or forcefull:
#DELETE all references to this release since we're updating it anyway.
myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check])
myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check])
release['AlbumTitle'] = unicode(releasedata['title'])
release['AlbumID'] = unicode(rgid)
release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None
release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None
release['ReleaseID'] = releasedata['id']
if 'release-group' not in releasedata:
raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid)
release['Type'] = unicode(releasedata['release-group']['type'])
if release['Type'] == 'Album' and 'secondary-type-list' in releasedata['release-group']:
secondary_type = unicode(releasedata['release-group']['secondary-type-list'][0])
if secondary_type != release['Type']:
release['Type'] = secondary_type
#making the assumption that the most important artist will be first in the list
if 'artist-credit' in releasedata:
release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id'])
release['ArtistName'] = unicode(releasedata['artist-credit-phrase'])
else:
logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.')
return False
release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown'
#assuming that the list will contain media and that the format will be consistent
try:
additional_medium=''
for position in releasedata['medium-list']:
if position['format'] == releasedata['medium-list'][0]['format']:
medium_count = int(position['position'])
else:
additional_medium = additional_medium+' + '+position['format']
if medium_count == 1:
disc_number = ''
else:
disc_number = str(medium_count)+'x'
packaged_medium = disc_number+releasedata['medium-list'][0]['format']+additional_medium
release['ReleaseFormat'] = unicode(packaged_medium)
except:
release['ReleaseFormat'] = u'Unknown'
release['Tracks'] = getTracksFromRelease(releasedata)
# What we're doing here now is first updating the allalbums & alltracks table to the most
# current info, then moving the appropriate release into the album table and its associated
# tracks into the tracks table
controlValueDict = {"ReleaseID" : release['ReleaseID']}
newValueDict = {"ArtistID": release['ArtistID'],
"ArtistName": release['ArtistName'],
"AlbumTitle": release['AlbumTitle'],
"AlbumID": release['AlbumID'],
"AlbumASIN": release['AlbumASIN'],
"ReleaseDate": release['ReleaseDate'],
"Type": release['Type'],
"ReleaseCountry": release['ReleaseCountry'],
"ReleaseFormat": release['ReleaseFormat']
}
myDB.upsert("allalbums", newValueDict, controlValueDict)
for track in release['Tracks']:
cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title'])
controlValueDict = {"TrackID": track['id'],
"ReleaseID": release['ReleaseID']}
newValueDict = {"ArtistID": release['ArtistID'],
"ArtistName": release['ArtistName'],
"AlbumTitle": release['AlbumTitle'],
"AlbumID": release['AlbumID'],
"AlbumASIN": release['AlbumASIN'],
"TrackTitle": track['title'],
"TrackDuration": track['duration'],
"TrackNumber": track['number'],
"CleanName": cleanname
}
match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
if not match:
match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone()
#if not match:
#match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
if match:
newValueDict['Location'] = match['Location']
newValueDict['BitRate'] = match['BitRate']
newValueDict['Format'] = match['Format']
#myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location']))
myDB.upsert("alltracks", newValueDict, controlValueDict)
num_new_releases = num_new_releases + 1
#print releasedata['title']
#print num_new_releases
if album_checker:
logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
else:
logger.info('[%s] New release %s (%s) added' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
if force_repackage1 == 1:
num_new_releases = -1
logger.info('[%s] Forcing repackage of %s, since dB releases have been removed' % (release['ArtistName'], release_title))
else:
num_new_releases = num_new_releases
return num_new_releases
def get_albums(artist_id):
""" Browses all releases of a given artist id, where release type is 'album'
For each release, if not alreay in the dictionary then the title and date are added to one dictionary, whilst title and release id are added to another.
For each release (album), its recordings (songs) are compared to those in words_dict2 (found previously by browsing the artist's recordings) -
where songs match their word counnt is found from words_dict2 and thus the average number of words in each album is calculated.
A bar chart of each album's year of release and the average number of words is plotted.
"""
limit = 100
offset = 0
releases_browse = musicbrainzngs.browse_releases(
release_type=['album'], artist=artist_id, limit=limit, offset=offset
) # Gets all recordings linked to artist - returns dict with keys ['recording-list', 'recording-count']
num_releases = releases_browse['release-count']
albums = [None] * num_releases
for page_num in range(num_releases // 100):
releases_browse = musicbrainzngs.browse_releases(
release_type=['album'],
artist=artist_id,
limit=limit,
offset=offset
) # Gets all recordings linked to artist - returns dict with keys ['recording-list', 'recording-count']
for position in range(limit): # 0 to 99
album_info = releases_browse['release-list'][position]
index = (page_num * 100) + position
albums[index] = album_info
offset += limit
releases_browse = musicbrainzngs.browse_releases(
release_type=['album'], artist=artist_id, limit=limit, offset=offset
) # Gets all recordings linked to artist - returns dict with keys ['recording-list', 'recording-count']
max_index = num_releases - ((num_releases // 100) * 100)
for position in range(max_index): # 0 to 99
album_info = releases_browse['release-list'][position]
index = ((num_releases // 100) * 100) + position
albums[index] = album_info
#albums = list(dict.fromkeys(albums))
album_id = {}
album_dates = {}
for album in albums:
if album['title'] not in album_id.keys(
) and album['title'] not in album_dates.keys():
try:
album_date = datetime.datetime.strptime(
str(album['date'])[:4], '%Y').date()
album_dates.update({album['title']: album_date.year})
album_id.update({album['title']: album['id']})
except KeyError:
pass
album_songs = {}
for album_name, release_id in album_id.items():
album_words = 0
album_num_songs = 0
recordings = musicbrainzngs.browse_recordings(
release=release_id, limit=100, offset=0
) # Gets all recordings linked to artist - returns dict with keys ['recording-list', 'recording-count']
for recording in recordings['recording-list']:
title = recording['title']
for key, value in words_dict2.items():
if key == title:
album_num_songs += 1
album_words += value
if album_num_songs != 0:
album_av = album_words / album_num_songs
else:
album_av = 0
album_songs.update({album_name: album_av})
album_name, av_words = zip(
*album_songs.items()) # unpack a list of pairs into two tuples
album_name, album_date = zip(
*album_dates.items()) # unpack a list of pairs into two tuples
plt.bar([str(date) for date in album_date], height=av_words)
plt.axhline(average,
linewidth=1,
color='m',
linestyle='--',
label="Average over all songs")
plt.xlabel("Year", fontsize=12)
plt.ylabel("Average number of words", fontsize=12)
plt.title(
"Graph of average number of words in album \n alongside year of release \n for {}"
.format(artist_name))
plt.xticks(rotation=90)
plt.legend()
plt.show()