def lastfm_fetch(count=30):
'''
fetch top artists and top tags for each artist, then save to artist_tag_store
retry several times if some tag data is not fetched (set number of retries in settings)
:param count: number of top artists to fetch
'''
artists = get_artists(count)
artist_names = [a['name'] for a in artists['artists']['artist']]
save_tags(artist_names)
retries = 0
incomplete = True
while retries <= max_retries and incomplete:
saved_artists = [
k.lstrip(ARTIST_PREFIX) for k in artist_tag_store.scan_iter()
]
missing_artists = list(set(artist_names) - set(saved_artists))
if not missing_artists:
incomplete = False
elif retries < max_retries:
logging.warn('Missing tag data for artists: %s' %
','.join(missing_artists))
save_tags(missing_artists)
retries = retries + 1
if incomplete:
logging.error('failed to get data for all artists')
else:
logging.info('success! we have tag data for all artists')
def lastfm_fetch (count=30):
'''
fetch top artists and top tags for each artist, then save to artist_tag_store
retry several times if some tag data is not fetched (set number of retries in settings)
:param count: number of top artists to fetch
'''
artists = get_artists(count)
artist_names = [a['name'] for a in artists['artists']['artist']]
save_tags(artist_names)
retries = 0
incomplete = True
while retries <= max_retries and incomplete:
saved_artists = [k.lstrip(ARTIST_PREFIX) for k in artist_tag_store.scan_iter()]
missing_artists = list(set(artist_names) - set(saved_artists))
if not missing_artists:
incomplete = False
elif retries < max_retries:
logging.warn('Missing tag data for artists: %s' % ','.join(missing_artists))
save_tags(missing_artists)
retries = retries + 1
if incomplete:
logging.error('failed to get data for all artists')
else:
logging.info('success! we have tag data for all artists')
def output_artist_tags():
'''
write artists and tags to a csv file specified in settings. format is artist,tag1;tag2;tag3
'''
with open(artist_tag_filename, 'wb') as csvfile:
w = csv.writer(csvfile)
w.writerow([s.encode('utf-8') for s in ['artist', 'tags']])
for key in artist_tag_store.scan_iter():
row = ([s.encode('utf-8') for s in [key.lstrip(ARTIST_PREFIX), ';'.join(artist_tag_store.lrange(key, 0, -1))]])
logging.debug(row)
w.writerow (row)
def get_artists_tags_graph():
'''
read artist tag data from artist tag store
returns a networkx bipartite graph with an edge between each artist and the top tags applied to them
'''
g = nx.Graph()
for artist in artist_tag_store.scan_iter():
artist_name = artist.lstrip(ARTIST_PREFIX)
g.add_node(('artist', artist_name), bipartite=ARTIST_MODE)
for tag in artist_tag_store.lrange(artist, 0, -1):
if not ('tag', tag) in g:
g.add_node(('tag', tag), bipartite=TAG_MODE)
g.add_edge(('artist', artist_name), ('tag', tag))
logging.info('artists-tags graph has %d nodes and %d edges' % (g.number_of_nodes(), g.number_of_edges()))
return g
def get_artists_tags_graph():
'''
read artist tag data from artist tag store
returns a networkx bipartite graph with an edge between each artist and the top tags applied to them
'''
g = nx.Graph()
for artist in artist_tag_store.scan_iter():
artist_name = artist.lstrip(ARTIST_PREFIX)
g.add_node(('artist', artist_name), bipartite=ARTIST_MODE)
for tag in artist_tag_store.lrange(artist, 0, -1):
if not ('tag', tag) in g:
g.add_node(('tag', tag), bipartite=TAG_MODE)
g.add_edge(('artist', artist_name), ('tag', tag))
logging.info('artists-tags graph has %d nodes and %d edges' %
(g.number_of_nodes(), g.number_of_edges()))
return g
def output_artist_tags():
'''
write artists and tags to a csv file specified in settings. format is artist,tag1;tag2;tag3
'''
with open(artist_tag_filename, 'wb') as csvfile:
w = csv.writer(csvfile)
w.writerow([s.encode('utf-8') for s in ['artist', 'tags']])
for key in artist_tag_store.scan_iter():
row = ([
s.encode('utf-8') for s in [
key.lstrip(ARTIST_PREFIX), ';'.join(
artist_tag_store.lrange(key, 0, -1))
]
])
logging.debug(row)
w.writerow(row)
