def exportScrobbles(): '''Fetch and process user scrobbles for the current year and for any year where exported json files are not present.''' def earliestScrobbleYear() -> int: '''Determine the earliest year for the user's scrobbles.''' lastPage = int( getReq(param=Param(method='user.getInfo')).get('playcount') ) - 100 # subtract 100 plays, in case some have "unknown" scrobble dates, i.e. 1970 return getReq( param=Param(method='user.getRecentTracks', lim=1, page=lastPage)).loc[0, 'date'].year param = Param(method='user.getRecentTracks', period='overall') currentYear = datetime.datetime.now().year for year in range(earliestScrobbleYear(), currentYear): paramYear = Param(method='user.getRecentTracks', period=year, fr=f'{year}-01-01 00:00:00', to=f'{year}-12-31 23:59:59') response = getReq(param=paramYear, collapse=False, limit=1) numPages = math.ceil( int(response.get('recenttracks').get('@attr').get('total')) / param.lim) if numPages != len(paramYear.filePath(glob='*json')): downloadData(paramYear, download=True) else: downloadData(paramYear, download=False) downloadData( Param(method='user.getRecentTracks', period=currentYear, fr=f'{currentYear}-01-01 00:00:00', to=f'{currentYear}-12-31 23:59:59')) mergeRecentTracks(param)
def earliestScrobbleYear() -> int: '''Determine the earliest year for the user's scrobbles.''' lastPage = int( getReq(param=Param(method='user.getInfo')).get('playcount') ) - 100 # subtract 100 plays, in case some have "unknown" scrobble dates, i.e. 1970 return getReq( param=Param(method='user.getRecentTracks', lim=1, page=lastPage)).loc[0, 'date'].year
def recommFromNeighbor(neighbor:str=None, method:str='user.getTopArtists', neighborThr:int=100, myThr:int=1000, **kwargs) -> pandas.DataFrame: '''Return neighbor's top artists/albums/songs missing from the user's top listens''' if not neighbor: return lastfmNeighbors() else: myData = loadUserData(Param(method=method)).head(myThr) param = Param(method=method, user=neighbor, lim=neighborThr) entity = param.splitMethod(lower=True, plural=False, strip=True) neighborData = getReq(param) cols = [f'{entity}_name', f'{entity}_playcount'] if entity != 'artist': cols = [f'artist_name', *cols] # numpy.setdiff1d(neighborData.get(f'{entity}_name'), myData.get(f'{entity}_name')) # [Compare and find missing strings in pandas Series](https://stackoverflow.com/a/58544291/13019084) return neighborData[[item not in myData.get(f'{entity}_name').to_list() for item in neighborData.get(f'{entity}_name').to_list()]][cols]
def downloadData(param: Param, download: bool = True): '''Download user data (if {download} is True) to json files, merge them into a flat pandas.DataFrame, and write it to disk.''' logging.info(f"{param.filePath().name.replace('.','|')}") if download: subMethod = param.splitMethod(lower=True) for f in param.filePath(glob='*json'): f.unlink() pbarManager = enlighten.get_manager() with pbarManager.counter(unit='page', leave=False) as pbar: while param.page <= param.nPages: fileName = param.filePath(ext=f'.{param.page:04d}.json') response = getReq(param=param, pbarManager=pbarManager, collapse=False) param.page = int( response.get(subMethod).get('@attr').get('page')) param.nPages = int( response.get(subMethod).get('@attr').get('totalPages')) pbar.total = param.nPages # [tqdm: update total without resetting time elapsed](https://stackoverflow.com/a/58961015/13019084) pbar.update() param.filePath().parent.mkdir(exist_ok=True) with open(file=fileName, mode='w') as jsonF: json.dump(obj=response, fp=jsonF) param.page += 1 time.sleep(param.sleep) pbarManager.stop() DF = loadJSON(param) df = flattenDF(param=param, DF=DF, writeToDisk=True) if param.splitMethod() in ['TopArtists', 'TopAlbums', 'TopTracks']: writeCSV(param=param, df=df)
def weeklyChart(method:str='user.getWeeklyArtistChart', weeksAgo:int=4, nWeeks:int=1, thr:int=6, **kwargs) -> pandas.DataFrame: '''Fetch data for {user.getWeekly*Chart} methods.''' (fr,to) = dateRange(weeksAgo=weeksAgo, nWeeks=nWeeks, **kwargs) param = Param(method=method, fr=fr, to=to) # ts = getReq(Param(method='user.getWeeklyChartList'))[::-1].reset_index() # [user.getWeeklyChartList](https://www.last.fm/api/show/user.getWeeklyChartList) # DF = getReq(param=Param(method=method, fr=ts.loc[weeksAgo+nWeeks,'list_from'], to=ts.loc[weeksAgo,'list_to'])) DF = getReq(param=param) return DF[DF[f'{param.splitMethod(plural=False, strip=True)}_playcount'] >= thr]
def recentDiscovery(entity:str='artist', weeksAgo:int=25, nWeeks:int=4, thr:int=10, **kwargs) -> List(str): '''Return artist/album/track_name listens that are not present in listening history before query period.''' (fr,to) = dateRange(weeksAgo=weeksAgo, nWeeks=nWeeks, **kwargs) param = Param(method='user.getRecentTracks', fr=fr, to=to) myData = loadUserData(param) query = myData[(myData.date_uts > toUnixTime(param.fr)) & (myData.date_uts < toUnixTime(param.to))].get(entity).value_counts() query = query[query >= thr].index beforeQuery = myData[(myData.date_uts < toUnixTime(param.fr))].get(entity).unique() return [item for item in query if item not in beforeQuery]
def forgottenAlbums(earlierYear:int=datetime.datetime.now().year-2, laterYear:int=datetime.datetime.now().year-1, minThr:int=50, maxThr:int=10) -> pandas.DataFrame: '''Return albums with many listens during {earlierYear} but few listens during {laterYear}.''' def filterAlbumScrobbles(scrobbles:pandas.DataFrame, year:int, thr:int, bound:str): yearScrobbles = scrobbles[(scrobbles.date >= f'{year}-01-01') & (scrobbles.date <= f'{year}-12-31')] yearAlbumFreq = yearScrobbles[['artist','album']].value_counts() # [Pandas: How to filter results of value_counts?](https://blog.softhints.com/pandas-how-to-filter-results-of-value_counts/) if bound == 'lower': yearAlbumFreq = yearAlbumFreq[yearAlbumFreq >= thr] if bound == 'upper': yearAlbumFreq = yearAlbumFreq[yearAlbumFreq <= thr] return yearAlbumFreq.rename(f'scrobbles{year}').reset_index() scrobbles = loadUserData(Param(method='user.getRecentTracks')) prev = filterAlbumScrobbles(scrobbles, year=earlierYear, thr=minThr, bound='lower') later = filterAlbumScrobbles(scrobbles, year=laterYear, thr=maxThr, bound='upper') return pandas.merge(prev, later, on=['artist','album'])
def friends(): return getReq(Param(method='user.getFriends', lim=10)) def getTopPersonalTag():
def lovedTracks(): return getReq(Param(method='user.getLovedTracks', lim=20)) def friends(): return getReq(Param(method='user.getFriends', lim=10))
def getArtistListeners(artist:str='opeth'): return int(getReq(Param(method='artist.getInfo'), artist=artist).get('stats').get('listeners'))
def getAlbumDuration(artist:str='opeth', album:str='damnation'): albumInfo = getReq(Param(method='album.getInfo'), artist=artist, album=album) return sum(int(track.get('duration')) for track in albumInfo.get('tracks').get('track')) if not 'error' in albumInfo.keys() else albumInfo
def getTopSimilarArtist(artist:str, **kwargs): topArtist = getReq(Param(method='user.getTopArtists', period='7day', lim=1)).loc[0,'artist_name'] if not artist else artist return getReq(Param(method='artist.getSimilar'), artist=topArtist, **kwargs)
def findDuplicateScrobbles(year:int=datetime.datetime.now().year, thr:int=600): recentTracks = loadUserData(Param(method='user.getRecentTracks', period=year)) return recentTracks[recentTracks.groupby('track_name')['date_uts'].diff().abs().fillna(thr+1) < thr] # [https://stackoverflow.com/a/44779167/13019084]
def main(): # downloadData(Param(method='user.getTopTracks', period='overall')) downloadData(Param(method='user.getTopAlbums', period='overall')) downloadData(Param(method='user.getTopArtists', period='overall')) exportScrobbles()
def getTopPersonalTag(): topTags = getReq(Param(method='user.getTopTags')) return getReq(param=Param(method='user.getPersonalTags'), tag=topTags.loc[0,'tag_name'], taggingtype='artist')
def trackScrobbles(artist:str='opeth', track:str='windowpane'): # [there is a new method user.getTrackScrobbles which is just like user.getArtistTracks, except also takes a "track" parameter](https://github.com/pylast/pylast/issues/298#issue-414708387) return getReq(Param(method='user.getTrackScrobbles', lim=20), artist=artist, track=track)
def getTopArtistTags(artist='opeth', **kwargs): return getReq(param=Param(method='artist.getTopTags'), artist=artist, **kwargs) def topArtistsCountry(country:str='spain', **kwargs): return getReq(param=Param(method='geo.getTopArtists', lim=10), country=country, **kwargs)
def earliestListen(query:str, entity:str='artist'): myData = loadUserData(Param(method='user.getRecentTracks')) matches = myData[myData.get(entity).str.contains(query, case=False)]
def topArtistsCountry(country:str='spain', **kwargs): return getReq(param=Param(method='geo.getTopArtists', lim=10), country=country, **kwargs) def getTopSimilarArtist(artist:str, **kwargs):