def process_one_user(user): """ Get recently played songs for this user and submit them to ListenBrainz. Args: user (spotify.Spotify): the user whose plays are to be imported. Raises: spotify.SpotifyAPIError: if we encounter errors from the Spotify API. spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying. or if we get errors while submitting the data to ListenBrainz """ if user.token_expired: try: user = spotify.refresh_user_token(user) except spotify.SpotifyAPIError: current_app.logger.error('Could not refresh user token from spotify', exc_info=True) raise listenbrainz_user = db_user.get(user.user_id) try: recently_played = get_user_recently_played(user) except (spotify.SpotifyListenBrainzError, spotify.SpotifyAPIError) as e: raise # convert listens to ListenBrainz format and validate them listens = [] if 'items' in recently_played: current_app.logger.debug('Received %d listens from Spotify for %s', len(recently_played['items']), str(user)) for item in recently_played['items']: listen = _convert_spotify_play_to_listen(item) try: validate_listen(listen, LISTEN_TYPE_IMPORT) listens.append(listen) except BadRequest: current_app.logger.error('Could not validate listen for user %s: %s', str(user), json.dumps(listen, indent=3), exc_info=True) # TODO: api_utils exposes werkzeug exceptions, if it's a more generic module it shouldn't be web-specific # if we don't have any new listens, return if len(listens) == 0: return latest_listened_at = max(listen['listened_at'] for listen in listens) # try to submit listens to ListenBrainz retries = 10 while retries >= 0: try: current_app.logger.debug('Submitting %d listens for user %s', len(listens), str(user)) insert_payload(listens, listenbrainz_user, listen_type=LISTEN_TYPE_IMPORT) current_app.logger.debug('Submitted!') break except (InternalServerError, ServiceUnavailable) as e: retries -= 1 current_app.logger.info('ISE while trying to import listens for %s: %s', str(user), str(e)) if retries == 0: raise spotify.SpotifyListenBrainzError('ISE while trying to import listens: %s', str(e)) # we've succeeded so update the last_updated field for this user spotify.update_latest_listened_at(user.user_id, latest_listened_at) spotify.update_last_updated(user.user_id)
def process_all_spotify_users(): """ Get a batch of users to be processed and import their Spotify plays. Returns: (success, failure) where success: the number of users whose plays were successfully imported. failure: the number of users for whom we faced errors while importing. """ try: users = spotify.get_active_users_to_process() except DatabaseException as e: current_app.logger.error('Cannot get list of users due to error %s', str(e), exc_info=True) return 0, 0 if not users: return 0, 0 current_app.logger.info('Process %d users...' % len(users)) success = 0 failure = 0 for u in users: t = time.time() current_app.logger.info('Importing spotify listens for user %s', str(u)) try: process_one_user(u) success += 1 except spotify.SpotifyAPIError as e: # if it is an error from the Spotify API, show the error message to the user spotify.update_last_updated( user_id=u.user_id, success=False, error_message=str(e), ) if not current_app.config['TESTING']: notify_error(u.musicbrainz_row_id, str(e)) failure += 1 except spotify.SpotifyListenBrainzError as e: current_app.logger.critical( 'spotify_reader could not import listens: %s', str(e), exc_info=True) failure += 1 except Exception as e: current_app.logger.critical( 'spotify_reader could not import listens: %s', str(e), exc_info=True) failure += 1 current_app.logger.info( 'Took a total of %.2f seconds to process user %s', time.time() - t, str(u)) current_app.logger.info('Processed %d users successfully!', success) current_app.logger.info('Encountered errors while processing %d users.', failure) return success, failure
def process_one_user(user): """ Get recently played songs for this user and submit them to ListenBrainz. Args: user (spotify.Spotify): the user whose plays are to be imported. Raises: spotify.SpotifyAPIError: if we encounter errors from the Spotify API. spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying. or if we get errors while submitting the data to ListenBrainz """ if user.token_expired: try: user = spotify.refresh_user_token(user) except spotify.SpotifyAPIError: current_app.logger.error( 'Could not refresh user token from spotify', exc_info=True) raise else: if user is None: current_app.logger.debug( "%s has revoked spotify authorization", str(user)) listenbrainz_user = db_user.get(user.user_id) currently_playing = get_user_currently_playing(user) if currently_playing is not None and 'item' in currently_playing: current_app.logger.debug('Received a currently playing track for %s', str(user)) listens = parse_and_validate_spotify_plays([currently_playing['item']], LISTEN_TYPE_PLAYING_NOW) submit_listens_to_listenbrainz(listenbrainz_user, listens, listen_type=LISTEN_TYPE_PLAYING_NOW) recently_played = get_user_recently_played(user) if recently_played is not None and 'items' in recently_played: listens = parse_and_validate_spotify_plays(recently_played['items'], LISTEN_TYPE_IMPORT) current_app.logger.debug('Received %d tracks for %s', len(listens), str(user)) # if we don't have any new listens, return if len(listens) == 0: return latest_listened_at = max(listen['listened_at'] for listen in listens) submit_listens_to_listenbrainz(listenbrainz_user, listens, listen_type=LISTEN_TYPE_IMPORT) # we've succeeded so update the last_updated field for this user spotify.update_latest_listened_at(user.user_id, latest_listened_at) spotify.update_last_updated(user.user_id) current_app.logger.info('imported %d listens for %s' % (len(listens), str(user)))
def process_one_user(user): """ Get recently played songs for this user and submit them to ListenBrainz. Args: user (spotify.Spotify): the user whose plays are to be imported. Raises: spotify.SpotifyAPIError: if we encounter errors from the Spotify API. spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying. or if we get errors while submitting the data to ListenBrainz """ if user.token_expired: user = spotify.refresh_user_token(user) listenbrainz_user = db_user.get(user.user_id) try: recently_played = get_user_recently_played(user) except (spotify.SpotifyListenBrainzError, spotify.SpotifyAPIError) as e: raise # convert listens to ListenBrainz format and validate them listens = [] if 'items' in recently_played: current_app.logger.debug('Received %d listens from Spotify for %s', len(recently_played['items']), str(user)) for item in recently_played['items']: listen = _convert_spotify_play_to_listen(item) try: validate_listen(listen, LISTEN_TYPE_IMPORT) listens.append(listen) except BadRequest: current_app.logger.error('Could not validate listen for user %s: %s', str(user), json.dumps(listen, indent=3), exc_info=True) # TODO: api_utils exposes werkzeug exceptions, if it's a more generic module it shouldn't be web-specific latest_listened_at = max(listen['listened_at'] for listen in listens) # try to submit listens to ListenBrainz retries = 10 while retries >= 0: try: current_app.logger.debug('Submitting %d listens for user %s', len(listens), str(user)) insert_payload(listens, listenbrainz_user, listen_type=LISTEN_TYPE_IMPORT) current_app.logger.debug('Submitted!') break except (InternalServerError, ServiceUnavailable) as e: retries -= 1 current_app.logger.info('ISE while trying to import listens for %s: %s', str(user), str(e)) if retries == 0: raise spotify.SpotifyListenBrainzError('ISE while trying to import listens: %s', str(e)) # we've succeeded so update the last_updated field for this user spotify.update_latest_listened_at(user.user_id, latest_listened_at) spotify.update_last_updated(user.user_id)
def process_all_spotify_users(): """ Get a batch of users to be processed and import their Spotify plays. Returns: (success, failure) where success: the number of users whose plays were successfully imported. failure: the number of users for whom we faced errors while importing. """ current_app.logger.info('Getting list of users to be processed...') try: users = spotify.get_active_users_to_process() except DatabaseException as e: current_app.logger.error('Cannot get list of users due to error %s', str(e), exc_info=True) return 0, 0 if not users: return 0, 0 success = 0 failure = 0 for u in users: t = time.time() current_app.logger.info('Importing spotify listens for user %s', str(u)) try: process_one_user(u) success += 1 except spotify.SpotifyAPIError as e: # if it is an error from the Spotify API, show the error message to the user spotify.update_last_updated( user_id=u.user_id, success=False, error_message=str(e), ) failure += 1 except spotify.SpotifyListenBrainzError as e: current_app.logger.critical('spotify_reader could not import listens: %s', str(e), exc_info=True) failure += 1 except Exception as e: current_app.logger.critical('spotify_reader could not import listens: %s', str(e), exc_info=True) failure += 1 current_app.logger.info('Took a total of %.2f seconds to process user %s', time.time() - t, str(u)) current_app.logger.info('Processed %d users successfully!', success) current_app.logger.info('Encountered errors while processing %d users.', failure) return success, failure
def process_one_user(user): """ Get recently played songs for this user and submit them to ListenBrainz. Args: user (spotify.Spotify): the user whose plays are to be imported. Raises: spotify.SpotifyAPIError: if we encounter errors from the Spotify API. spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying. or if we get errors while submitting the data to ListenBrainz """ try: if user.token_expired: user = spotify.refresh_user_token(user) listenbrainz_user = db_user.get(user.user_id) # If there is no playback, currently_playing will be None. # There are two playing types, track and episode. We use only the # track type. Therefore, when the user's playback type is not a track, # Spotify will set the item field to null which becomes None after # parsing the JSON. Due to these reasons, we cannot simplify the # checks below. currently_playing = get_user_currently_playing(user) if currently_playing is not None: currently_playing_item = currently_playing.get('item', None) if currently_playing_item is not None: current_app.logger.debug( 'Received a currently playing track for %s', str(user)) listens = parse_and_validate_spotify_plays( [currently_playing_item], LISTEN_TYPE_PLAYING_NOW) if listens: submit_listens_to_listenbrainz( listenbrainz_user, listens, listen_type=LISTEN_TYPE_PLAYING_NOW) recently_played = get_user_recently_played(user) if recently_played is not None and 'items' in recently_played: listens = parse_and_validate_spotify_plays( recently_played['items'], LISTEN_TYPE_IMPORT) current_app.logger.debug('Received %d tracks for %s', len(listens), str(user)) # if we don't have any new listens, return if len(listens) == 0: return latest_listened_at = max(listen['listened_at'] for listen in listens) submit_listens_to_listenbrainz(listenbrainz_user, listens, listen_type=LISTEN_TYPE_IMPORT) # we've succeeded so update the last_updated field for this user spotify.update_latest_listened_at(user.user_id, latest_listened_at) spotify.update_last_updated(user.user_id) current_app.logger.info('imported %d listens for %s' % (len(listens), str(user))) except spotify.SpotifyAPIError as e: # if it is an error from the Spotify API, show the error message to the user spotify.update_last_updated( user_id=user.user_id, success=False, error_message=str(e), ) if not current_app.config['TESTING']: notify_error(user.musicbrainz_row_id, str(e)) raise spotify.SpotifyListenBrainzError( "Could not refresh user token from spotify") except spotify.SpotifyInvalidGrantError: if not current_app.config['TESTING']: notify_error( user.musicbrainz_row_id, "It seems like you've revoked permission for us to read your spotify account" ) # user has revoked authorization through spotify ui or deleted their spotify account etc. # in any of these cases, we should delete user from our spotify db as well. db_spotify.delete_spotify(user.user_id) raise spotify.SpotifyListenBrainzError( "User has revoked spotify authorization")