def test_refresh_user_token_bad(self, mock_get_spotify_oauth):
mock_oauth = MagicMock()
mock_oauth.refresh_access_token.return_value = None
mock_get_spotify_oauth.return_value = mock_oauth
with self.assertRaises(spotify.SpotifyAPIError):
spotify.refresh_user_token(self.spotify_user)
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
if user.token_expired:
try:
user = spotify.refresh_user_token(user)
except spotify.SpotifyAPIError:
current_app.logger.error('Could not refresh user token from spotify', exc_info=True)
raise
listenbrainz_user = db_user.get(user.user_id)
try:
recently_played = get_user_recently_played(user)
except (spotify.SpotifyListenBrainzError, spotify.SpotifyAPIError) as e:
raise
# convert listens to ListenBrainz format and validate them
listens = []
if 'items' in recently_played:
current_app.logger.debug('Received %d listens from Spotify for %s', len(recently_played['items']), str(user))
for item in recently_played['items']:
listen = _convert_spotify_play_to_listen(item)
try:
validate_listen(listen, LISTEN_TYPE_IMPORT)
listens.append(listen)
except BadRequest:
current_app.logger.error('Could not validate listen for user %s: %s', str(user), json.dumps(listen, indent=3), exc_info=True)
# TODO: api_utils exposes werkzeug exceptions, if it's a more generic module it shouldn't be web-specific
# if we don't have any new listens, return
if len(listens) == 0:
return
latest_listened_at = max(listen['listened_at'] for listen in listens)
# try to submit listens to ListenBrainz
retries = 10
while retries >= 0:
try:
current_app.logger.debug('Submitting %d listens for user %s', len(listens), str(user))
insert_payload(listens, listenbrainz_user, listen_type=LISTEN_TYPE_IMPORT)
current_app.logger.debug('Submitted!')
break
except (InternalServerError, ServiceUnavailable) as e:
retries -= 1
current_app.logger.info('ISE while trying to import listens for %s: %s', str(user), str(e))
if retries == 0:
raise spotify.SpotifyListenBrainzError('ISE while trying to import listens: %s', str(e))
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
def test_refresh_user_token(self, mock_update_token, mock_get_spotify_oauth, mock_get_user):
expires_at = int(time.time()) + 3600
mock_get_spotify_oauth.return_value.refresh_access_token.return_value = {
'access_token': 'tokentoken',
'refresh_token': 'refreshtokentoken',
'expires_at': expires_at,
}
spotify.refresh_user_token(self.spotify_user)
mock_update_token.assert_called_with(
self.spotify_user.user_id,
'tokentoken',
'refreshtokentoken',
expires_at,
)
mock_get_user.assert_called_with(self.spotify_user.user_id)
def test_refresh_user_token(self, mock_update_token,
mock_get_spotify_oauth, mock_get_user):
expires_at = int(time.time()) + 3600
mock_get_spotify_oauth.return_value.refresh_access_token.return_value = {
'access_token': 'tokentoken',
'refresh_token': 'refreshtokentoken',
'expires_at': expires_at,
'scope': '',
}
spotify.refresh_user_token(self.spotify_user)
mock_update_token.assert_called_with(
self.spotify_user.user_id,
'tokentoken',
'refreshtokentoken',
expires_at,
)
mock_get_user.assert_called_with(self.spotify_user.user_id)
def test_refresh_user_token_only_access(self, mock_requests,
mock_update_token, mock_get_user):
expires_at = int(time.time()) + 3600
mock_requests.post(spotify.OAUTH_TOKEN_URL,
status_code=200,
json={
'access_token': 'tokentoken',
'expires_in': 3600,
'scope': '',
})
spotify.refresh_user_token(self.spotify_user)
mock_update_token.assert_called_with(
self.spotify_user.user_id,
'tokentoken',
'old-refresh-token',
expires_at,
)
mock_get_user.assert_called_with(self.spotify_user.user_id)
def test_refresh_user_token_only_access(self, mock_requests,
mock_update_token, mock_get_user):
mock_requests.post(spotify.OAUTH_TOKEN_URL,
status_code=200,
json={
'access_token': 'tokentoken',
'expires_in': 3600,
'scope': '',
})
spotify.refresh_user_token(self.spotify_user)
mock_update_token.assert_called_with(
self.spotify_user.user_id,
'tokentoken',
'old-refresh-token',
mock.
ANY # expires_at cannot be accurately calculated hence using mock.ANY
# another option is using a range for expires_at and a Matcher but that seems far more work
)
mock_get_user.assert_called_with(self.spotify_user.user_id)
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
if user.token_expired:
try:
user = spotify.refresh_user_token(user)
except spotify.SpotifyAPIError:
current_app.logger.error(
'Could not refresh user token from spotify', exc_info=True)
raise
listenbrainz_user = db_user.get(user.user_id)
currently_playing = get_user_currently_playing(user)
if currently_playing is not None and 'item' in currently_playing:
current_app.logger.debug('Received a currently playing track for %s',
str(user))
listens = parse_and_validate_spotify_plays([currently_playing['item']],
LISTEN_TYPE_PLAYING_NOW)
submit_listens_to_listenbrainz(listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_PLAYING_NOW)
recently_played = get_user_recently_played(user)
if recently_played is not None and 'items' in recently_played:
listens = parse_and_validate_spotify_plays(recently_played['items'],
LISTEN_TYPE_IMPORT)
current_app.logger.debug('Received %d tracks for %s', len(listens),
str(user))
# if we don't have any new listens, return
if len(listens) == 0:
return
latest_listened_at = max(listen['listened_at'] for listen in listens)
submit_listens_to_listenbrainz(listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_IMPORT)
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
current_app.logger.info('imported %d listens for %s' %
(len(listens), str(user)))
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
if user.token_expired:
user = spotify.refresh_user_token(user)
listenbrainz_user = db_user.get(user.user_id)
try:
recently_played = get_user_recently_played(user)
except (spotify.SpotifyListenBrainzError, spotify.SpotifyAPIError) as e:
raise
# convert listens to ListenBrainz format and validate them
listens = []
if 'items' in recently_played:
current_app.logger.debug('Received %d listens from Spotify for %s', len(recently_played['items']), str(user))
for item in recently_played['items']:
listen = _convert_spotify_play_to_listen(item)
try:
validate_listen(listen, LISTEN_TYPE_IMPORT)
listens.append(listen)
except BadRequest:
current_app.logger.error('Could not validate listen for user %s: %s', str(user), json.dumps(listen, indent=3), exc_info=True)
# TODO: api_utils exposes werkzeug exceptions, if it's a more generic module it shouldn't be web-specific
latest_listened_at = max(listen['listened_at'] for listen in listens)
# try to submit listens to ListenBrainz
retries = 10
while retries >= 0:
try:
current_app.logger.debug('Submitting %d listens for user %s', len(listens), str(user))
insert_payload(listens, listenbrainz_user, listen_type=LISTEN_TYPE_IMPORT)
current_app.logger.debug('Submitted!')
break
except (InternalServerError, ServiceUnavailable) as e:
retries -= 1
current_app.logger.info('ISE while trying to import listens for %s: %s', str(user), str(e))
if retries == 0:
raise spotify.SpotifyListenBrainzError('ISE while trying to import listens: %s', str(e))
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
def refresh_spotify_token():
spotify_user = spotify.get_user(current_user.id)
if not spotify_user:
raise APINotFound("User has not authenticated to Spotify")
if spotify_user.token_expired:
try:
spotify_user = spotify.refresh_user_token(spotify_user)
except spotify.SpotifyAPIError:
raise APIServiceUnavailable("Cannot refresh Spotify token right now")
return jsonify({
'id': current_user.id,
'musicbrainz_id': current_user.musicbrainz_id,
'user_token': spotify_user.user_token,
'permission': spotify_user.permission,
})
def get_user_recently_played(user):
""" Get recently played songs from Spotify for specified user.
This uses the 'me/player/recently-played' endpoint, which only allows us to get the last 50 plays
for one user.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Returns:
the response from the spotify API consisting of the list of recently played songs.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
"""
retries = 10
delay = 1
tried_to_refresh_token = False
while retries > 0:
try:
recently_played = user.get_spotipy_client()._get("me/player/recently-played", limit=50)
break
except SpotifyException as e:
retries -= 1
if e.http_status == 429:
# Rate Limit Problems -- the client handles these, but it can still give up
# after a certain number of retries, so we look at the header and try the
# request again, if the error is raised
time_to_sleep = e.headers.get('Retry-After', delay)
current_app.logger.warn('Encountered a rate limit, sleeping %d seconds and trying again...', time_to_sleep)
time.sleep(time_to_sleep)
delay += 1
if retries == 0:
raise spotify.SpotifyListenBrainzError('Encountered a rate limit.')
elif e.http_status in (400, 403, 404):
current_app.logger.critical('Error from the Spotify API for user %s: %s', str(user), str(e), exc_info=True)
raise spotify.SpotifyAPIError('Error from the Spotify API while getting listens: %s', str(e))
elif e.http_status >= 500 and e.http_status < 600:
# these errors are not our fault, most probably. so just log them and retry.
current_app.logger.error('Error while trying to get listens for user %s: %s', str(user), str(e), exc_info=True)
if retries == 0:
raise spotify.SpotifyAPIError('Error from the spotify API while getting listens: %s', str(e))
elif e.http_status == 401:
# if we get 401 Unauthorized from Spotify, that means our token might have expired.
# In that case, try to refresh the token, if there is an error even while refreshing
# give up and report to the user.
# We only try to refresh the token once, if we still get 401 after that, we give up.
if not tried_to_refresh_token:
try:
user = spotify.refresh_user_token(user)
except SpotifyError as err:
raise spotify.SpotifyAPIError('Could not authenticate with Spotify, please unlink and link your account again.')
tried_to_refresh_token = True
else:
raise spotify.SpotifyAPIError('Could not authenticate with Spotify, please unlink and link your account again.')
except Exception as e:
retries -= 1
current_app.logger.error('Unexpected error while getting listens: %s', str(e), exc_info=True)
if retries == 0:
raise spotify.SpotifyListenBrainzError('Unexpected error while getting listens: %s' % str(e))
return recently_played
def get_user_recently_played(user):
""" Get recently played songs from Spotify for specified user.
This uses the 'me/player/recently-played' endpoint, which only allows us to get the last 50 plays
for one user.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Returns:
the response from the spotify API consisting of the list of recently played songs.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
"""
retries = 10
delay = 1
tried_to_refresh_token = False
while retries > 0:
try:
recently_played = user.get_spotipy_client()._get("me/player/recently-played", limit=50)
break
except SpotifyException as e:
retries -= 1
if e.http_status == 429:
# Rate Limit Problems -- the client handles these, but it can still give up
# after a certain number of retries, so we look at the header and try the
# request again, if the error is raised
time_to_sleep = e.headers.get('Retry-After', delay)
current_app.logger.warn('Encountered a rate limit, sleeping %d seconds and trying again...', time_to_sleep)
time.sleep(time_to_sleep)
delay += 1
if retries == 0:
raise spotify.SpotifyListenBrainzError('Encountered a rate limit.')
elif e.http_status in (400, 403, 404):
current_app.logger.critical('Error from the Spotify API for user %s: %s', str(user), str(e), exc_info=True)
raise spotify.SpotifyAPIError('Error from the Spotify API while getting listens: %s', str(e))
elif e.http_status >= 500 and e.http_status < 600:
# these errors are not our fault, most probably. so just log them and retry.
current_app.logger.error('Error while trying to get listens for user %s: %s', str(user), str(e), exc_info=True)
if retries == 0:
raise spotify.SpotifyAPIError('Error from the spotify API while getting listens: %s', str(e))
elif e.http_status == 401:
# if we get 401 Unauthorized from Spotify, that means our token might have expired.
# In that case, try to refresh the token, if there is an error even while refreshing
# give up and report to the user.
# We only try to refresh the token once, if we still get 401 after that, we give up.
if not tried_to_refresh_token:
try:
user = spotify.refresh_user_token(user)
except SpotifyError as err:
raise spotify.SpotifyAPIError('Could not authenticate with Spotify, please unlink and link your account again.')
tried_to_refresh_token = True
else:
raise spotify.SpotifyAPIError('Could not authenticate with Spotify, please unlink and link your account again.')
except Exception as e:
retries -= 1
current_app.logger.error('Unexpected error while getting listens: %s', str(e), exc_info=True)
if retries == 0:
raise spotify.SpotifyListenBrainzError('Unexpected error while getting listens: %s' % str(e))
return recently_played
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
try:
if user.token_expired:
user = spotify.refresh_user_token(user)
listenbrainz_user = db_user.get(user.user_id)
# If there is no playback, currently_playing will be None.
# There are two playing types, track and episode. We use only the
# track type. Therefore, when the user's playback type is not a track,
# Spotify will set the item field to null which becomes None after
# parsing the JSON. Due to these reasons, we cannot simplify the
# checks below.
currently_playing = get_user_currently_playing(user)
if currently_playing is not None:
currently_playing_item = currently_playing.get('item', None)
if currently_playing_item is not None:
current_app.logger.debug(
'Received a currently playing track for %s', str(user))
listens = parse_and_validate_spotify_plays(
[currently_playing_item], LISTEN_TYPE_PLAYING_NOW)
if listens:
submit_listens_to_listenbrainz(
listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_PLAYING_NOW)
recently_played = get_user_recently_played(user)
if recently_played is not None and 'items' in recently_played:
listens = parse_and_validate_spotify_plays(
recently_played['items'], LISTEN_TYPE_IMPORT)
current_app.logger.debug('Received %d tracks for %s', len(listens),
str(user))
# if we don't have any new listens, return
if len(listens) == 0:
return
latest_listened_at = max(listen['listened_at'] for listen in listens)
submit_listens_to_listenbrainz(listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_IMPORT)
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
current_app.logger.info('imported %d listens for %s' %
(len(listens), str(user)))
except spotify.SpotifyAPIError as e:
# if it is an error from the Spotify API, show the error message to the user
spotify.update_last_updated(
user_id=user.user_id,
success=False,
error_message=str(e),
)
if not current_app.config['TESTING']:
notify_error(user.musicbrainz_row_id, str(e))
raise spotify.SpotifyListenBrainzError(
"Could not refresh user token from spotify")
except spotify.SpotifyInvalidGrantError:
if not current_app.config['TESTING']:
notify_error(
user.musicbrainz_row_id,
"It seems like you've revoked permission for us to read your spotify account"
)
# user has revoked authorization through spotify ui or deleted their spotify account etc.
# in any of these cases, we should delete user from our spotify db as well.
db_spotify.delete_spotify(user.user_id)
raise spotify.SpotifyListenBrainzError(
"User has revoked spotify authorization")
def make_api_request(user, spotipy_call, **kwargs):
""" Make an request to the Spotify API for particular user at specified endpoint with args.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
spotipy_call (function): the Spotipy function which makes request to the required API endpoint
Returns:
the response from the spotify API
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
"""
retries = 10
delay = 1
tried_to_refresh_token = False
while retries > 0:
try:
recently_played = spotipy_call(**kwargs)
break
except SpotifyException as e:
retries -= 1
if e.http_status == 429:
# Rate Limit Problems -- the client handles these, but it can still give up
# after a certain number of retries, so we look at the header and try the
# request again, if the error is raised
try:
time_to_sleep = int(e.headers.get('Retry-After', delay))
except ValueError:
time_to_sleep = delay
current_app.logger.warn(
'Encountered a rate limit, sleeping %d seconds and trying again...',
time_to_sleep)
time.sleep(time_to_sleep)
delay += 1
if retries == 0:
raise spotify.SpotifyListenBrainzError(
'Encountered a rate limit.')
elif e.http_status in (400, 403):
current_app.logger.critical(
'Error from the Spotify API for user %s: %s',
str(user),
str(e),
exc_info=True)
raise spotify.SpotifyAPIError(
'Error from the Spotify API while getting listens: %s',
str(e))
elif e.http_status >= 500 and e.http_status < 600:
# these errors are not our fault, most probably. so just log them and retry.
current_app.logger.error(
'Error while trying to get listens for user %s: %s',
str(user),
str(e),
exc_info=True)
if retries == 0:
raise spotify.SpotifyAPIError(
'Error from the spotify API while getting listens: %s',
str(e))
elif e.http_status == 401:
# if we get 401 Unauthorized from Spotify, that means our token might have expired.
# In that case, try to refresh the token, if there is an error even while refreshing
# give up and report to the user.
# We only try to refresh the token once, if we still get 401 after that, we give up.
if not tried_to_refresh_token:
user = spotify.refresh_user_token(user)
tried_to_refresh_token = True
else:
raise spotify.SpotifyAPIError(
'Could not authenticate with Spotify, please unlink and link your account again.'
)
elif e.http_status == 404:
current_app.logger.error(
"404 while trying to get listens for user %s",
str(user),
exc_info=True)
if retries == 0:
raise spotify.SpotifyListenBrainzError(
"404 while trying to get listens for user %s" %
str(user))
except Exception as e:
retries -= 1
current_app.logger.error(
'Unexpected error while getting listens: %s',
str(e),
exc_info=True)
if retries == 0:
raise spotify.SpotifyListenBrainzError(
'Unexpected error while getting listens: %s' % str(e))
return recently_played
