def lastfm_fetch(count=30): ''' fetch top artists and top tags for each artist, then save to artist_tag_store retry several times if some tag data is not fetched (set number of retries in settings) :param count: number of top artists to fetch ''' artists = get_artists(count) artist_names = [a['name'] for a in artists['artists']['artist']] save_tags(artist_names) retries = 0 incomplete = True while retries <= max_retries and incomplete: saved_artists = [ k.lstrip(ARTIST_PREFIX) for k in artist_tag_store.scan_iter() ] missing_artists = list(set(artist_names) - set(saved_artists)) if not missing_artists: incomplete = False elif retries < max_retries: logging.warn('Missing tag data for artists: %s' % ','.join(missing_artists)) save_tags(missing_artists) retries = retries + 1 if incomplete: logging.error('failed to get data for all artists') else: logging.info('success! we have tag data for all artists')
def lastfm_fetch (count=30): ''' fetch top artists and top tags for each artist, then save to artist_tag_store retry several times if some tag data is not fetched (set number of retries in settings) :param count: number of top artists to fetch ''' artists = get_artists(count) artist_names = [a['name'] for a in artists['artists']['artist']] save_tags(artist_names) retries = 0 incomplete = True while retries <= max_retries and incomplete: saved_artists = [k.lstrip(ARTIST_PREFIX) for k in artist_tag_store.scan_iter()] missing_artists = list(set(artist_names) - set(saved_artists)) if not missing_artists: incomplete = False elif retries < max_retries: logging.warn('Missing tag data for artists: %s' % ','.join(missing_artists)) save_tags(missing_artists) retries = retries + 1 if incomplete: logging.error('failed to get data for all artists') else: logging.info('success! we have tag data for all artists')
def output_artist_tags(): ''' write artists and tags to a csv file specified in settings. format is artist,tag1;tag2;tag3 ''' with open(artist_tag_filename, 'wb') as csvfile: w = csv.writer(csvfile) w.writerow([s.encode('utf-8') for s in ['artist', 'tags']]) for key in artist_tag_store.scan_iter(): row = ([s.encode('utf-8') for s in [key.lstrip(ARTIST_PREFIX), ';'.join(artist_tag_store.lrange(key, 0, -1))]]) logging.debug(row) w.writerow (row)
def get_artists_tags_graph(): ''' read artist tag data from artist tag store returns a networkx bipartite graph with an edge between each artist and the top tags applied to them ''' g = nx.Graph() for artist in artist_tag_store.scan_iter(): artist_name = artist.lstrip(ARTIST_PREFIX) g.add_node(('artist', artist_name), bipartite=ARTIST_MODE) for tag in artist_tag_store.lrange(artist, 0, -1): if not ('tag', tag) in g: g.add_node(('tag', tag), bipartite=TAG_MODE) g.add_edge(('artist', artist_name), ('tag', tag)) logging.info('artists-tags graph has %d nodes and %d edges' % (g.number_of_nodes(), g.number_of_edges())) return g
def output_artist_tags(): ''' write artists and tags to a csv file specified in settings. format is artist,tag1;tag2;tag3 ''' with open(artist_tag_filename, 'wb') as csvfile: w = csv.writer(csvfile) w.writerow([s.encode('utf-8') for s in ['artist', 'tags']]) for key in artist_tag_store.scan_iter(): row = ([ s.encode('utf-8') for s in [ key.lstrip(ARTIST_PREFIX), ';'.join( artist_tag_store.lrange(key, 0, -1)) ] ]) logging.debug(row) w.writerow(row)