def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
if user.token_expired:
try:
user = spotify.refresh_user_token(user)
except spotify.SpotifyAPIError:
current_app.logger.error('Could not refresh user token from spotify', exc_info=True)
raise
listenbrainz_user = db_user.get(user.user_id)
try:
recently_played = get_user_recently_played(user)
except (spotify.SpotifyListenBrainzError, spotify.SpotifyAPIError) as e:
raise
# convert listens to ListenBrainz format and validate them
listens = []
if 'items' in recently_played:
current_app.logger.debug('Received %d listens from Spotify for %s', len(recently_played['items']), str(user))
for item in recently_played['items']:
listen = _convert_spotify_play_to_listen(item)
try:
validate_listen(listen, LISTEN_TYPE_IMPORT)
listens.append(listen)
except BadRequest:
current_app.logger.error('Could not validate listen for user %s: %s', str(user), json.dumps(listen, indent=3), exc_info=True)
# TODO: api_utils exposes werkzeug exceptions, if it's a more generic module it shouldn't be web-specific
# if we don't have any new listens, return
if len(listens) == 0:
return
latest_listened_at = max(listen['listened_at'] for listen in listens)
# try to submit listens to ListenBrainz
retries = 10
while retries >= 0:
try:
current_app.logger.debug('Submitting %d listens for user %s', len(listens), str(user))
insert_payload(listens, listenbrainz_user, listen_type=LISTEN_TYPE_IMPORT)
current_app.logger.debug('Submitted!')
break
except (InternalServerError, ServiceUnavailable) as e:
retries -= 1
current_app.logger.info('ISE while trying to import listens for %s: %s', str(user), str(e))
if retries == 0:
raise spotify.SpotifyListenBrainzError('ISE while trying to import listens: %s', str(e))
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
def process_all_spotify_users():
""" Get a batch of users to be processed and import their Spotify plays.
Returns:
(success, failure) where
success: the number of users whose plays were successfully imported.
failure: the number of users for whom we faced errors while importing.
"""
try:
users = spotify.get_active_users_to_process()
except DatabaseException as e:
current_app.logger.error('Cannot get list of users due to error %s',
str(e),
exc_info=True)
return 0, 0
if not users:
return 0, 0
current_app.logger.info('Process %d users...' % len(users))
success = 0
failure = 0
for u in users:
t = time.time()
current_app.logger.info('Importing spotify listens for user %s',
str(u))
try:
process_one_user(u)
success += 1
except spotify.SpotifyAPIError as e:
# if it is an error from the Spotify API, show the error message to the user
spotify.update_last_updated(
user_id=u.user_id,
success=False,
error_message=str(e),
)
if not current_app.config['TESTING']:
notify_error(u.musicbrainz_row_id, str(e))
failure += 1
except spotify.SpotifyListenBrainzError as e:
current_app.logger.critical(
'spotify_reader could not import listens: %s',
str(e),
exc_info=True)
failure += 1
except Exception as e:
current_app.logger.critical(
'spotify_reader could not import listens: %s',
str(e),
exc_info=True)
failure += 1
current_app.logger.info(
'Took a total of %.2f seconds to process user %s',
time.time() - t, str(u))
current_app.logger.info('Processed %d users successfully!', success)
current_app.logger.info('Encountered errors while processing %d users.',
failure)
return success, failure
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
if user.token_expired:
try:
user = spotify.refresh_user_token(user)
except spotify.SpotifyAPIError:
current_app.logger.error(
'Could not refresh user token from spotify', exc_info=True)
raise
else:
if user is None:
current_app.logger.debug(
"%s has revoked spotify authorization", str(user))
listenbrainz_user = db_user.get(user.user_id)
currently_playing = get_user_currently_playing(user)
if currently_playing is not None and 'item' in currently_playing:
current_app.logger.debug('Received a currently playing track for %s',
str(user))
listens = parse_and_validate_spotify_plays([currently_playing['item']],
LISTEN_TYPE_PLAYING_NOW)
submit_listens_to_listenbrainz(listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_PLAYING_NOW)
recently_played = get_user_recently_played(user)
if recently_played is not None and 'items' in recently_played:
listens = parse_and_validate_spotify_plays(recently_played['items'],
LISTEN_TYPE_IMPORT)
current_app.logger.debug('Received %d tracks for %s', len(listens),
str(user))
# if we don't have any new listens, return
if len(listens) == 0:
return
latest_listened_at = max(listen['listened_at'] for listen in listens)
submit_listens_to_listenbrainz(listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_IMPORT)
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
current_app.logger.info('imported %d listens for %s' %
(len(listens), str(user)))
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
if user.token_expired:
user = spotify.refresh_user_token(user)
listenbrainz_user = db_user.get(user.user_id)
try:
recently_played = get_user_recently_played(user)
except (spotify.SpotifyListenBrainzError, spotify.SpotifyAPIError) as e:
raise
# convert listens to ListenBrainz format and validate them
listens = []
if 'items' in recently_played:
current_app.logger.debug('Received %d listens from Spotify for %s', len(recently_played['items']), str(user))
for item in recently_played['items']:
listen = _convert_spotify_play_to_listen(item)
try:
validate_listen(listen, LISTEN_TYPE_IMPORT)
listens.append(listen)
except BadRequest:
current_app.logger.error('Could not validate listen for user %s: %s', str(user), json.dumps(listen, indent=3), exc_info=True)
# TODO: api_utils exposes werkzeug exceptions, if it's a more generic module it shouldn't be web-specific
latest_listened_at = max(listen['listened_at'] for listen in listens)
# try to submit listens to ListenBrainz
retries = 10
while retries >= 0:
try:
current_app.logger.debug('Submitting %d listens for user %s', len(listens), str(user))
insert_payload(listens, listenbrainz_user, listen_type=LISTEN_TYPE_IMPORT)
current_app.logger.debug('Submitted!')
break
except (InternalServerError, ServiceUnavailable) as e:
retries -= 1
current_app.logger.info('ISE while trying to import listens for %s: %s', str(user), str(e))
if retries == 0:
raise spotify.SpotifyListenBrainzError('ISE while trying to import listens: %s', str(e))
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
def process_all_spotify_users():
""" Get a batch of users to be processed and import their Spotify plays.
Returns:
(success, failure) where
success: the number of users whose plays were successfully imported.
failure: the number of users for whom we faced errors while importing.
"""
current_app.logger.info('Getting list of users to be processed...')
try:
users = spotify.get_active_users_to_process()
except DatabaseException as e:
current_app.logger.error('Cannot get list of users due to error %s', str(e), exc_info=True)
return 0, 0
if not users:
return 0, 0
success = 0
failure = 0
for u in users:
t = time.time()
current_app.logger.info('Importing spotify listens for user %s', str(u))
try:
process_one_user(u)
success += 1
except spotify.SpotifyAPIError as e:
# if it is an error from the Spotify API, show the error message to the user
spotify.update_last_updated(
user_id=u.user_id,
success=False,
error_message=str(e),
)
failure += 1
except spotify.SpotifyListenBrainzError as e:
current_app.logger.critical('spotify_reader could not import listens: %s', str(e), exc_info=True)
failure += 1
except Exception as e:
current_app.logger.critical('spotify_reader could not import listens: %s', str(e), exc_info=True)
failure += 1
current_app.logger.info('Took a total of %.2f seconds to process user %s', time.time() - t, str(u))
current_app.logger.info('Processed %d users successfully!', success)
current_app.logger.info('Encountered errors while processing %d users.', failure)
return success, failure
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
try:
if user.token_expired:
user = spotify.refresh_user_token(user)
listenbrainz_user = db_user.get(user.user_id)
# If there is no playback, currently_playing will be None.
# There are two playing types, track and episode. We use only the
# track type. Therefore, when the user's playback type is not a track,
# Spotify will set the item field to null which becomes None after
# parsing the JSON. Due to these reasons, we cannot simplify the
# checks below.
currently_playing = get_user_currently_playing(user)
if currently_playing is not None:
currently_playing_item = currently_playing.get('item', None)
if currently_playing_item is not None:
current_app.logger.debug(
'Received a currently playing track for %s', str(user))
listens = parse_and_validate_spotify_plays(
[currently_playing_item], LISTEN_TYPE_PLAYING_NOW)
if listens:
submit_listens_to_listenbrainz(
listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_PLAYING_NOW)
recently_played = get_user_recently_played(user)
if recently_played is not None and 'items' in recently_played:
listens = parse_and_validate_spotify_plays(
recently_played['items'], LISTEN_TYPE_IMPORT)
current_app.logger.debug('Received %d tracks for %s', len(listens),
str(user))
# if we don't have any new listens, return
if len(listens) == 0:
return
latest_listened_at = max(listen['listened_at'] for listen in listens)
submit_listens_to_listenbrainz(listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_IMPORT)
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
current_app.logger.info('imported %d listens for %s' %
(len(listens), str(user)))
except spotify.SpotifyAPIError as e:
# if it is an error from the Spotify API, show the error message to the user
spotify.update_last_updated(
user_id=user.user_id,
success=False,
error_message=str(e),
)
if not current_app.config['TESTING']:
notify_error(user.musicbrainz_row_id, str(e))
raise spotify.SpotifyListenBrainzError(
"Could not refresh user token from spotify")
except spotify.SpotifyInvalidGrantError:
if not current_app.config['TESTING']:
notify_error(
user.musicbrainz_row_id,
"It seems like you've revoked permission for us to read your spotify account"
)
# user has revoked authorization through spotify ui or deleted their spotify account etc.
# in any of these cases, we should delete user from our spotify db as well.
db_spotify.delete_spotify(user.user_id)
raise spotify.SpotifyListenBrainzError(
"User has revoked spotify authorization")
