def submit_listens_to_listenbrainz(listenbrainz_user, listens, listen_type=LISTEN_TYPE_IMPORT): """ Submit a batch of listens to ListenBrainz Args: listenbrainz_user (dict): the user whose listens are to be submitted listens (list): a list of listens to be submitted listen_type: the type of listen (single, import, playing_now) """ username = listenbrainz_user['musicbrainz_id'] retries = 10 while retries >= 0: try: current_app.logger.debug('Submitting %d listens for user %s', len(listens), username) insert_payload(listens, listenbrainz_user, listen_type=listen_type) current_app.logger.debug('Submitted!') break except (InternalServerError, ServiceUnavailable) as e: retries -= 1 current_app.logger.error( 'ISE while trying to import listens for %s: %s', username, str(e)) if retries == 0: raise spotify.SpotifyListenBrainzError( 'ISE while trying to import listens: %s', str(e))
def process_one_user(user): """ Get recently played songs for this user and submit them to ListenBrainz. Args: user (spotify.Spotify): the user whose plays are to be imported. Raises: spotify.SpotifyAPIError: if we encounter errors from the Spotify API. spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying. or if we get errors while submitting the data to ListenBrainz """ if user.token_expired: user = spotify.refresh_user_token(user) listenbrainz_user = db_user.get(user.user_id) try: recently_played = get_user_recently_played(user) except (spotify.SpotifyListenBrainzError, spotify.SpotifyAPIError) as e: raise # convert listens to ListenBrainz format and validate them listens = [] if 'items' in recently_played: current_app.logger.debug('Received %d listens from Spotify for %s', len(recently_played['items']), str(user)) for item in recently_played['items']: listen = _convert_spotify_play_to_listen(item) try: validate_listen(listen, LISTEN_TYPE_IMPORT) listens.append(listen) except BadRequest: current_app.logger.error('Could not validate listen for user %s: %s', str(user), json.dumps(listen, indent=3), exc_info=True) # TODO: api_utils exposes werkzeug exceptions, if it's a more generic module it shouldn't be web-specific latest_listened_at = max(listen['listened_at'] for listen in listens) # try to submit listens to ListenBrainz retries = 10 while retries >= 0: try: current_app.logger.debug('Submitting %d listens for user %s', len(listens), str(user)) insert_payload(listens, listenbrainz_user, listen_type=LISTEN_TYPE_IMPORT) current_app.logger.debug('Submitted!') break except (InternalServerError, ServiceUnavailable) as e: retries -= 1 current_app.logger.info('ISE while trying to import listens for %s: %s', str(user), str(e)) if retries == 0: raise spotify.SpotifyListenBrainzError('ISE while trying to import listens: %s', str(e)) # we've succeeded so update the last_updated field for this user spotify.update_latest_listened_at(user.user_id, latest_listened_at) spotify.update_last_updated(user.user_id)
def get_user_recently_played(user): """ Get recently played songs from Spotify for specified user. This uses the 'me/player/recently-played' endpoint, which only allows us to get the last 50 plays for one user. Args: user (spotify.Spotify): the user whose plays are to be imported. Returns: the response from the spotify API consisting of the list of recently played songs. Raises: spotify.SpotifyAPIError: if we encounter errors from the Spotify API. spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying. """ retries = 10 delay = 1 tried_to_refresh_token = False while retries > 0: try: recently_played = user.get_spotipy_client()._get("me/player/recently-played", limit=50) break except SpotifyException as e: retries -= 1 if e.http_status == 429: # Rate Limit Problems -- the client handles these, but it can still give up # after a certain number of retries, so we look at the header and try the # request again, if the error is raised time_to_sleep = e.headers.get('Retry-After', delay) current_app.logger.warn('Encountered a rate limit, sleeping %d seconds and trying again...', time_to_sleep) time.sleep(time_to_sleep) delay += 1 if retries == 0: raise spotify.SpotifyListenBrainzError('Encountered a rate limit.') elif e.http_status in (400, 403, 404): current_app.logger.critical('Error from the Spotify API for user %s: %s', str(user), str(e), exc_info=True) raise spotify.SpotifyAPIError('Error from the Spotify API while getting listens: %s', str(e)) elif e.http_status >= 500 and e.http_status < 600: # these errors are not our fault, most probably. so just log them and retry. current_app.logger.error('Error while trying to get listens for user %s: %s', str(user), str(e), exc_info=True) if retries == 0: raise spotify.SpotifyAPIError('Error from the spotify API while getting listens: %s', str(e)) elif e.http_status == 401: # if we get 401 Unauthorized from Spotify, that means our token might have expired. # In that case, try to refresh the token, if there is an error even while refreshing # give up and report to the user. # We only try to refresh the token once, if we still get 401 after that, we give up. if not tried_to_refresh_token: try: user = spotify.refresh_user_token(user) except SpotifyError as err: raise spotify.SpotifyAPIError('Could not authenticate with Spotify, please unlink and link your account again.') tried_to_refresh_token = True else: raise spotify.SpotifyAPIError('Could not authenticate with Spotify, please unlink and link your account again.') except Exception as e: retries -= 1 current_app.logger.error('Unexpected error while getting listens: %s', str(e), exc_info=True) if retries == 0: raise spotify.SpotifyListenBrainzError('Unexpected error while getting listens: %s' % str(e)) return recently_played
def process_one_user(user): """ Get recently played songs for this user and submit them to ListenBrainz. Args: user (spotify.Spotify): the user whose plays are to be imported. Raises: spotify.SpotifyAPIError: if we encounter errors from the Spotify API. spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying. or if we get errors while submitting the data to ListenBrainz """ try: if user.token_expired: user = spotify.refresh_user_token(user) listenbrainz_user = db_user.get(user.user_id) # If there is no playback, currently_playing will be None. # There are two playing types, track and episode. We use only the # track type. Therefore, when the user's playback type is not a track, # Spotify will set the item field to null which becomes None after # parsing the JSON. Due to these reasons, we cannot simplify the # checks below. currently_playing = get_user_currently_playing(user) if currently_playing is not None: currently_playing_item = currently_playing.get('item', None) if currently_playing_item is not None: current_app.logger.debug( 'Received a currently playing track for %s', str(user)) listens = parse_and_validate_spotify_plays( [currently_playing_item], LISTEN_TYPE_PLAYING_NOW) if listens: submit_listens_to_listenbrainz( listenbrainz_user, listens, listen_type=LISTEN_TYPE_PLAYING_NOW) recently_played = get_user_recently_played(user) if recently_played is not None and 'items' in recently_played: listens = parse_and_validate_spotify_plays( recently_played['items'], LISTEN_TYPE_IMPORT) current_app.logger.debug('Received %d tracks for %s', len(listens), str(user)) # if we don't have any new listens, return if len(listens) == 0: return latest_listened_at = max(listen['listened_at'] for listen in listens) submit_listens_to_listenbrainz(listenbrainz_user, listens, listen_type=LISTEN_TYPE_IMPORT) # we've succeeded so update the last_updated field for this user spotify.update_latest_listened_at(user.user_id, latest_listened_at) spotify.update_last_updated(user.user_id) current_app.logger.info('imported %d listens for %s' % (len(listens), str(user))) except spotify.SpotifyAPIError as e: # if it is an error from the Spotify API, show the error message to the user spotify.update_last_updated( user_id=user.user_id, success=False, error_message=str(e), ) if not current_app.config['TESTING']: notify_error(user.musicbrainz_row_id, str(e)) raise spotify.SpotifyListenBrainzError( "Could not refresh user token from spotify") except spotify.SpotifyInvalidGrantError: if not current_app.config['TESTING']: notify_error( user.musicbrainz_row_id, "It seems like you've revoked permission for us to read your spotify account" ) # user has revoked authorization through spotify ui or deleted their spotify account etc. # in any of these cases, we should delete user from our spotify db as well. db_spotify.delete_spotify(user.user_id) raise spotify.SpotifyListenBrainzError( "User has revoked spotify authorization")
def make_api_request(user, spotipy_call, **kwargs): """ Make an request to the Spotify API for particular user at specified endpoint with args. Args: user (spotify.Spotify): the user whose plays are to be imported. spotipy_call (function): the Spotipy function which makes request to the required API endpoint Returns: the response from the spotify API Raises: spotify.SpotifyAPIError: if we encounter errors from the Spotify API. spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying. """ retries = 10 delay = 1 tried_to_refresh_token = False while retries > 0: try: recently_played = spotipy_call(**kwargs) break except SpotifyException as e: retries -= 1 if e.http_status == 429: # Rate Limit Problems -- the client handles these, but it can still give up # after a certain number of retries, so we look at the header and try the # request again, if the error is raised try: time_to_sleep = int(e.headers.get('Retry-After', delay)) except ValueError: time_to_sleep = delay current_app.logger.warn( 'Encountered a rate limit, sleeping %d seconds and trying again...', time_to_sleep) time.sleep(time_to_sleep) delay += 1 if retries == 0: raise spotify.SpotifyListenBrainzError( 'Encountered a rate limit.') elif e.http_status in (400, 403): current_app.logger.critical( 'Error from the Spotify API for user %s: %s', str(user), str(e), exc_info=True) raise spotify.SpotifyAPIError( 'Error from the Spotify API while getting listens: %s', str(e)) elif e.http_status >= 500 and e.http_status < 600: # these errors are not our fault, most probably. so just log them and retry. current_app.logger.error( 'Error while trying to get listens for user %s: %s', str(user), str(e), exc_info=True) if retries == 0: raise spotify.SpotifyAPIError( 'Error from the spotify API while getting listens: %s', str(e)) elif e.http_status == 401: # if we get 401 Unauthorized from Spotify, that means our token might have expired. # In that case, try to refresh the token, if there is an error even while refreshing # give up and report to the user. # We only try to refresh the token once, if we still get 401 after that, we give up. if not tried_to_refresh_token: user = spotify.refresh_user_token(user) tried_to_refresh_token = True else: raise spotify.SpotifyAPIError( 'Could not authenticate with Spotify, please unlink and link your account again.' ) elif e.http_status == 404: current_app.logger.error( "404 while trying to get listens for user %s", str(user), exc_info=True) if retries == 0: raise spotify.SpotifyListenBrainzError( "404 while trying to get listens for user %s" % str(user)) except Exception as e: retries -= 1 current_app.logger.error( 'Unexpected error while getting listens: %s', str(e), exc_info=True) if retries == 0: raise spotify.SpotifyListenBrainzError( 'Unexpected error while getting listens: %s' % str(e)) return recently_played