def exportScrobbles():
'''Fetch and process user scrobbles for the current year and for any year where exported json files are not present.'''
def earliestScrobbleYear() -> int:
'''Determine the earliest year for the user's scrobbles.'''
lastPage = int(
getReq(param=Param(method='user.getInfo')).get('playcount')
) - 100 # subtract 100 plays, in case some have "unknown" scrobble dates, i.e. 1970
return getReq(
param=Param(method='user.getRecentTracks', lim=1,
page=lastPage)).loc[0, 'date'].year
param = Param(method='user.getRecentTracks', period='overall')
currentYear = datetime.datetime.now().year
for year in range(earliestScrobbleYear(), currentYear):
paramYear = Param(method='user.getRecentTracks',
period=year,
fr=f'{year}-01-01 00:00:00',
to=f'{year}-12-31 23:59:59')
response = getReq(param=paramYear, collapse=False, limit=1)
numPages = math.ceil(
int(response.get('recenttracks').get('@attr').get('total')) /
param.lim)
if numPages != len(paramYear.filePath(glob='*json')):
downloadData(paramYear, download=True)
else:
downloadData(paramYear, download=False)
downloadData(
Param(method='user.getRecentTracks',
period=currentYear,
fr=f'{currentYear}-01-01 00:00:00',
to=f'{currentYear}-12-31 23:59:59'))
mergeRecentTracks(param)
def earliestScrobbleYear() -> int:
'''Determine the earliest year for the user's scrobbles.'''
lastPage = int(
getReq(param=Param(method='user.getInfo')).get('playcount')
) - 100 # subtract 100 plays, in case some have "unknown" scrobble dates, i.e. 1970
return getReq(
param=Param(method='user.getRecentTracks', lim=1,
page=lastPage)).loc[0, 'date'].year
def recommFromNeighbor(neighbor:str=None, method:str='user.getTopArtists', neighborThr:int=100, myThr:int=1000, **kwargs) -> pandas.DataFrame:
'''Return neighbor's top artists/albums/songs missing from the user's top listens'''
if not neighbor: return lastfmNeighbors()
else:
myData = loadUserData(Param(method=method)).head(myThr)
param = Param(method=method, user=neighbor, lim=neighborThr)
entity = param.splitMethod(lower=True, plural=False, strip=True)
neighborData = getReq(param)
cols = [f'{entity}_name', f'{entity}_playcount']
if entity != 'artist': cols = [f'artist_name', *cols]
# numpy.setdiff1d(neighborData.get(f'{entity}_name'), myData.get(f'{entity}_name')) # [Compare and find missing strings in pandas Series](https://stackoverflow.com/a/58544291/13019084)
return neighborData[[item not in myData.get(f'{entity}_name').to_list() for item in neighborData.get(f'{entity}_name').to_list()]][cols]
def downloadData(param: Param, download: bool = True):
'''Download user data (if {download} is True) to json files, merge them into a flat pandas.DataFrame, and write it to disk.'''
logging.info(f"{param.filePath().name.replace('.','|')}")
if download:
subMethod = param.splitMethod(lower=True)
for f in param.filePath(glob='*json'):
f.unlink()
pbarManager = enlighten.get_manager()
with pbarManager.counter(unit='page', leave=False) as pbar:
while param.page <= param.nPages:
fileName = param.filePath(ext=f'.{param.page:04d}.json')
response = getReq(param=param,
pbarManager=pbarManager,
collapse=False)
param.page = int(
response.get(subMethod).get('@attr').get('page'))
param.nPages = int(
response.get(subMethod).get('@attr').get('totalPages'))
pbar.total = param.nPages # [tqdm: update total without resetting time elapsed](https://stackoverflow.com/a/58961015/13019084)
pbar.update()
param.filePath().parent.mkdir(exist_ok=True)
with open(file=fileName, mode='w') as jsonF:
json.dump(obj=response, fp=jsonF)
param.page += 1
time.sleep(param.sleep)
pbarManager.stop()
DF = loadJSON(param)
df = flattenDF(param=param, DF=DF, writeToDisk=True)
if param.splitMethod() in ['TopArtists', 'TopAlbums', 'TopTracks']:
writeCSV(param=param, df=df)
def weeklyChart(method:str='user.getWeeklyArtistChart', weeksAgo:int=4, nWeeks:int=1, thr:int=6, **kwargs) -> pandas.DataFrame:
'''Fetch data for {user.getWeekly*Chart} methods.'''
(fr,to) = dateRange(weeksAgo=weeksAgo, nWeeks=nWeeks, **kwargs)
param = Param(method=method, fr=fr, to=to)
# ts = getReq(Param(method='user.getWeeklyChartList'))[::-1].reset_index() # [user.getWeeklyChartList](https://www.last.fm/api/show/user.getWeeklyChartList)
# DF = getReq(param=Param(method=method, fr=ts.loc[weeksAgo+nWeeks,'list_from'], to=ts.loc[weeksAgo,'list_to']))
DF = getReq(param=param)
return DF[DF[f'{param.splitMethod(plural=False, strip=True)}_playcount'] >= thr]
def recentDiscovery(entity:str='artist', weeksAgo:int=25, nWeeks:int=4, thr:int=10, **kwargs) -> List(str):
'''Return artist/album/track_name listens that are not present in listening history before query period.'''
(fr,to) = dateRange(weeksAgo=weeksAgo, nWeeks=nWeeks, **kwargs)
param = Param(method='user.getRecentTracks', fr=fr, to=to)
myData = loadUserData(param)
query = myData[(myData.date_uts > toUnixTime(param.fr)) & (myData.date_uts < toUnixTime(param.to))].get(entity).value_counts()
query = query[query >= thr].index
beforeQuery = myData[(myData.date_uts < toUnixTime(param.fr))].get(entity).unique()
return [item for item in query if item not in beforeQuery]
def forgottenAlbums(earlierYear:int=datetime.datetime.now().year-2, laterYear:int=datetime.datetime.now().year-1, minThr:int=50, maxThr:int=10) -> pandas.DataFrame:
'''Return albums with many listens during {earlierYear} but few listens during {laterYear}.'''
def filterAlbumScrobbles(scrobbles:pandas.DataFrame, year:int, thr:int, bound:str):
yearScrobbles = scrobbles[(scrobbles.date >= f'{year}-01-01') & (scrobbles.date <= f'{year}-12-31')]
yearAlbumFreq = yearScrobbles[['artist','album']].value_counts() # [Pandas: How to filter results of value_counts?](https://blog.softhints.com/pandas-how-to-filter-results-of-value_counts/)
if bound == 'lower': yearAlbumFreq = yearAlbumFreq[yearAlbumFreq >= thr]
if bound == 'upper': yearAlbumFreq = yearAlbumFreq[yearAlbumFreq <= thr]
return yearAlbumFreq.rename(f'scrobbles{year}').reset_index()
scrobbles = loadUserData(Param(method='user.getRecentTracks'))
prev = filterAlbumScrobbles(scrobbles, year=earlierYear, thr=minThr, bound='lower')
later = filterAlbumScrobbles(scrobbles, year=laterYear, thr=maxThr, bound='upper')
return pandas.merge(prev, later, on=['artist','album'])
def friends(): return getReq(Param(method='user.getFriends', lim=10)) def getTopPersonalTag():
def lovedTracks(): return getReq(Param(method='user.getLovedTracks', lim=20)) def friends(): return getReq(Param(method='user.getFriends', lim=10))
def getArtistListeners(artist:str='opeth'):
return int(getReq(Param(method='artist.getInfo'), artist=artist).get('stats').get('listeners'))
def getAlbumDuration(artist:str='opeth', album:str='damnation'):
albumInfo = getReq(Param(method='album.getInfo'), artist=artist, album=album)
return sum(int(track.get('duration')) for track in albumInfo.get('tracks').get('track')) if not 'error' in albumInfo.keys() else albumInfo
def getTopSimilarArtist(artist:str, **kwargs):
topArtist = getReq(Param(method='user.getTopArtists', period='7day', lim=1)).loc[0,'artist_name'] if not artist else artist
return getReq(Param(method='artist.getSimilar'), artist=topArtist, **kwargs)
def findDuplicateScrobbles(year:int=datetime.datetime.now().year, thr:int=600):
recentTracks = loadUserData(Param(method='user.getRecentTracks', period=year))
return recentTracks[recentTracks.groupby('track_name')['date_uts'].diff().abs().fillna(thr+1) < thr] # [https://stackoverflow.com/a/44779167/13019084]
def main():
# downloadData(Param(method='user.getTopTracks', period='overall'))
downloadData(Param(method='user.getTopAlbums', period='overall'))
downloadData(Param(method='user.getTopArtists', period='overall'))
exportScrobbles()
def getTopPersonalTag():
topTags = getReq(Param(method='user.getTopTags'))
return getReq(param=Param(method='user.getPersonalTags'), tag=topTags.loc[0,'tag_name'], taggingtype='artist')
def trackScrobbles(artist:str='opeth', track:str='windowpane'): # [there is a new method user.getTrackScrobbles which is just like user.getArtistTracks, except also takes a "track" parameter](https://github.com/pylast/pylast/issues/298#issue-414708387)
return getReq(Param(method='user.getTrackScrobbles', lim=20), artist=artist, track=track)
def getTopArtistTags(artist='opeth', **kwargs): return getReq(param=Param(method='artist.getTopTags'), artist=artist, **kwargs) def topArtistsCountry(country:str='spain', **kwargs): return getReq(param=Param(method='geo.getTopArtists', lim=10), country=country, **kwargs)
def earliestListen(query:str, entity:str='artist'):
myData = loadUserData(Param(method='user.getRecentTracks'))
matches = myData[myData.get(entity).str.contains(query, case=False)]
def topArtistsCountry(country:str='spain', **kwargs): return getReq(param=Param(method='geo.getTopArtists', lim=10), country=country, **kwargs) def getTopSimilarArtist(artist:str, **kwargs):
