def _playlist_resultset_to_model(connection, result, load_recordings):
"""Parse the result of an sql query to get playlists
Fill in related data (username, created_for username) and collaborators
"""
playlists = []
user_id_map = {}
for row in result:
creator_id = row["creator_id"]
if creator_id not in user_id_map:
# TODO: Do this lookup in bulk
user_id_map[creator_id] = db_user.get(creator_id)
created_for_id = row["created_for_id"]
if created_for_id and created_for_id not in user_id_map:
user_id_map[created_for_id] = db_user.get(created_for_id)
row = dict(row)
row["creator"] = user_id_map[creator_id]["musicbrainz_id"]
if created_for_id:
row["created_for"] = user_id_map[created_for_id]["musicbrainz_id"]
row["recordings"] = []
playlist = model_playlist.Playlist.parse_obj(row)
playlists.append(playlist)
playlist_ids = [p.id for p in playlists]
if playlist_ids:
if load_recordings:
playlist_recordings = get_recordings_for_playlists(connection, playlist_ids)
for p in playlists:
p.recordings = playlist_recordings.get(p.id, [])
playlist_collaborator_ids = get_collaborators_for_playlists(connection, playlist_ids)
for p in playlists:
p.collaborator_ids = playlist_collaborator_ids.get(p.id, [])
p.collaborators = get_collaborators_names_from_ids(p.collaborator_ids)
return playlists
def get_by_mbid(
playlist_id: str,
load_recordings: bool = True) -> Optional[model_playlist.Playlist]:
"""Get a playlist given its mbid
Arguments:
playlist_id: the uuid of a playlist to get
load_recordings: If true, load the recordings for the playlist too
Raises:
ValueError: if playlist_id isn't a valid uuid
Returns:
a ``model_playlist.Playlist``, or ``None`` if no such Playlist with the
given id exists
"""
query = sqlalchemy.text("""
SELECT pl.id
, pl.mbid
, pl.creator_id
, pl.name
, pl.description
, pl.public
, pl.created
, pl.last_updated
, pl.copied_from_id
, pl.created_for_id
, pl.algorithm_metadata
, copy.mbid as copied_from_mbid
FROM playlist.playlist AS pl
LEFT JOIN playlist.playlist AS copy
ON pl.copied_from_id = copy.id
WHERE pl.mbid = :mbid""")
with ts.engine.connect() as connection:
result = connection.execute(query, {"mbid": playlist_id})
obj = result.fetchone()
if not obj:
return None
obj = dict(obj)
user_id = obj['creator_id']
user = db_user.get(user_id)
obj['creator'] = user['musicbrainz_id']
if obj['created_for_id']:
created_for_user = db_user.get(obj['created_for_id'])
if created_for_user:
obj['created_for'] = created_for_user['musicbrainz_id']
if load_recordings:
playlist_map = get_recordings_for_playlists(
connection, [obj['id']])
obj['recordings'] = playlist_map.get(obj['id'], [])
else:
obj['recordings'] = []
playlist_collaborator_ids = get_collaborators_for_playlists(
connection, [obj['id']])
collaborator_ids_list = playlist_collaborator_ids.get(obj['id'], [])
obj['collaborator_ids'] = collaborator_ids_list
obj['collaborators'] = get_collaborators_names_from_ids(
collaborator_ids_list)
return model_playlist.Playlist.parse_obj(obj)
def setUp(self):
super(TestPostgresListenStore, self).setUp()
self.log = logging.getLogger(__name__)
self.logstore = init_postgres_connection(self.config.SQLALCHEMY_DATABASE_URI)
# TODO: Does this use the normal or test DB??
self.testuser_id = db_user.create("test")
user = db_user.get(self.testuser_id)
print(user)
self.testuser_name = db_user.get(self.testuser_id)['musicbrainz_id']
def test_delete_when_spotify_import_activated(self):
user_id = db_user.create(11, 'kishore')
user = db_user.get(user_id)
self.assertIsNotNone(user)
db_spotify.create_spotify(user_id, 'user token', 'refresh token', 0)
db_user.delete(user_id)
user = db_user.get(user_id)
self.assertIsNone(user)
token = db_spotify.get_token_for_user(user_id)
self.assertIsNone(token)
def test_delete_when_spotify_import_activated(self):
user_id = db_user.create(11, 'kishore')
user = db_user.get(user_id)
self.assertIsNotNone(user)
db_oauth.save_token(user_id, ExternalServiceType.SPOTIFY, 'user token',
'refresh token', 0, True,
['user-read-recently-played'])
db_user.delete(user_id)
user = db_user.get(user_id)
self.assertIsNone(user)
token = db_oauth.get_token(user_id, ExternalServiceType.SPOTIFY)
self.assertIsNone(token)
def setUp(self):
super(TestInfluxListenStore, self).setUp()
self.log = logging.getLogger(__name__)
self.logstore = init_influx_connection(
self.log, {
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
})
self.testuser_id = db_user.create("test")
user = db_user.get(self.testuser_id)
print(user)
self.testuser_name = db_user.get(self.testuser_id)['musicbrainz_id']
def test_ts_filters(self, timescale):
"""Check that max_ts and min_ts are passed to timescale """
user = User.from_dbrow(db_user.get(1)).to_dict()
timescale.return_value = ([], 0, 0)
# If no parameter is given, use current time as the to_ts
self.client.get(url_for('user.profile', user_name='iliekcomputers'))
req_call = mock.call(user, limit=25, from_ts=None)
timescale.assert_has_calls([req_call])
timescale.reset_mock()
# max_ts query param -> to_ts timescale param
self.client.get(url_for('user.profile', user_name='iliekcomputers'),
query_string={'max_ts': 1520946000})
req_call = mock.call(user, limit=25, to_ts=1520946000)
timescale.assert_has_calls([req_call])
timescale.reset_mock()
# min_ts query param -> from_ts timescale param
self.client.get(url_for('user.profile', user_name='iliekcomputers'),
query_string={'min_ts': 1520941000})
req_call = mock.call(user, limit=25, from_ts=1520941000)
timescale.assert_has_calls([req_call])
timescale.reset_mock()
# If max_ts and min_ts set, only max_ts is used
self.client.get(url_for('user.profile', user_name='iliekcomputers'),
query_string={
'min_ts': 1520941000,
'max_ts': 1520946000
})
req_call = mock.call(user, limit=25, to_ts=1520946000)
timescale.assert_has_calls([req_call])
def setUp(self):
super(TestAPICompatUserClass, self).setUp()
self.log = logging.getLogger(__name__)
self.logstore = init_influx_connection(
self.log, {
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
})
# Create a user
uid = db_user.create("test_api_compat_user")
self.assertIsNotNone(db_user.get(uid))
with db.engine.connect() as connection:
result = connection.execute(
text("""
SELECT *
FROM "user"
WHERE id = :id
"""), {
"id": uid,
})
row = result.fetchone()
self.user = User(row['id'], row['created'], row['musicbrainz_id'],
row['auth_token'])
# Insert some listens
date = datetime(2015, 9, 3, 0, 0, 0)
self.log.info("Inserting test data...")
test_data = generate_data(date, 100, self.user.name)
self.logstore.insert(test_data)
self.log.info("Test data inserted")
def submit_listens():
""" Submit listens received from clients into the listenstore after validating them. """
try:
session = _get_session(request.form.get('s', ''))
except BadRequest:
return 'BADSESSION\n', 401
listens = []
index = 0
while True:
listen = _to_native_api(request.form, append_key='[{}]'.format(index))
if listen is None:
break
else:
listens.append(listen)
index += 1
if not listens:
return 'FAILED Invalid data submitted!\n', 400
user = db_user.get(session.user_id)
insert_payload(listens, user)
return 'OK\n'
def get_recordings_for_playlists(connection, playlist_ids: List[int]):
""""""
query = sqlalchemy.text("""
SELECT id
, playlist_id
, position
, mbid
, added_by_id
, created
FROM playlist.playlist_recording
WHERE playlist_id IN :playlist_ids
ORDER BY playlist_id, position
""")
result = connection.execute(query, {"playlist_ids": tuple(playlist_ids)})
user_id_map = {}
playlist_recordings_map = collections.defaultdict(list)
for row in result:
added_by_id = row["added_by_id"]
if added_by_id not in user_id_map:
# TODO: Do this lookup in bulk
user_id_map[added_by_id] = db_user.get(added_by_id)
row = dict(row)
row["added_by"] = user_id_map[added_by_id]["musicbrainz_id"]
playlist_recording = model_playlist.PlaylistRecording.parse_obj(row)
playlist_recordings_map[playlist_recording.playlist_id].append(playlist_recording)
for playlist_id in playlist_ids:
if playlist_id not in playlist_recordings_map:
playlist_recordings_map[playlist_id] = []
return dict(playlist_recordings_map)
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
if user.token_expired:
try:
user = spotify.refresh_user_token(user)
except spotify.SpotifyAPIError:
current_app.logger.error('Could not refresh user token from spotify', exc_info=True)
raise
listenbrainz_user = db_user.get(user.user_id)
try:
recently_played = get_user_recently_played(user)
except (spotify.SpotifyListenBrainzError, spotify.SpotifyAPIError) as e:
raise
# convert listens to ListenBrainz format and validate them
listens = []
if 'items' in recently_played:
current_app.logger.debug('Received %d listens from Spotify for %s', len(recently_played['items']), str(user))
for item in recently_played['items']:
listen = _convert_spotify_play_to_listen(item)
try:
validate_listen(listen, LISTEN_TYPE_IMPORT)
listens.append(listen)
except BadRequest:
current_app.logger.error('Could not validate listen for user %s: %s', str(user), json.dumps(listen, indent=3), exc_info=True)
# TODO: api_utils exposes werkzeug exceptions, if it's a more generic module it shouldn't be web-specific
# if we don't have any new listens, return
if len(listens) == 0:
return
latest_listened_at = max(listen['listened_at'] for listen in listens)
# try to submit listens to ListenBrainz
retries = 10
while retries >= 0:
try:
current_app.logger.debug('Submitting %d listens for user %s', len(listens), str(user))
insert_payload(listens, listenbrainz_user, listen_type=LISTEN_TYPE_IMPORT)
current_app.logger.debug('Submitted!')
break
except (InternalServerError, ServiceUnavailable) as e:
retries -= 1
current_app.logger.info('ISE while trying to import listens for %s: %s', str(user), str(e))
if retries == 0:
raise spotify.SpotifyListenBrainzError('ISE while trying to import listens: %s', str(e))
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
def submit_listens():
""" Submit listens received from clients into the listenstore after validating them. """
try:
session = _get_session(request.form.get('s', ''))
except BadRequest:
return 'BADSESSION\n', 401
listens = []
index = 0
while True:
listen = _to_native_api(request.form, append_key='[{}]'.format(index))
if listen is None:
break
else:
listens.append(listen)
index += 1
if not listens:
return 'FAILED Invalid data submitted!\n', 400
user = db_user.get(session.user_id)
insert_payload(listens, user)
return 'OK\n'
def setUp(self):
super(TestAPICompatUserClass, self).setUp()
self.log = logging.getLogger(__name__)
self.logstore = init_influx_connection(
self.log, {
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
})
# Create a user
uid = db_user.create("test_api_compat_user")
self.assertIsNotNone(db_user.get(uid))
with db.engine.connect() as connection:
result = connection.execute(
text("""
SELECT *
FROM "user"
WHERE id = :id
"""), {
"id": uid,
})
row = result.fetchone()
self.user = User(row['id'], row['created'], row['musicbrainz_id'],
row['auth_token'])
def update_playlist(playlist: model_playlist.Playlist):
"""Update playlist metadata (Name, description, public flag)
Arguments:
"""
query = sqlalchemy.text("""
UPDATE playlist.playlist
SET name = :name
, description = :description
, public = :public
WHERE id = :id
""")
with ts.engine.connect() as connection:
params = playlist.dict(include={'id', 'name', 'description', 'public'})
connection.execute(query, params)
# Unconditionally add collaborators, this allows us to delete all collaborators
# if [] is passed in.
# TODO: Optimise this by getting collaborators from the database and only updating
# if what has passed is different to what exists
add_playlist_collaborators(connection, playlist.id, playlist.collaborator_ids)
collaborator_ids = get_collaborators_for_playlists(connection, [playlist.id])
collaborator_ids = collaborator_ids.get(playlist.id, [])
collaborators = []
# TODO: Look this up in one query
for user_id in collaborator_ids:
user = db_user.get(user_id)
if user:
collaborators.append(user["musicbrainz_id"])
playlist.collaborators = collaborators
playlist.last_updated = set_last_updated(connection, playlist.id)
return playlist
def setUp(self):
super(TestAPICompatUserClass, self).setUp()
self.log = logging.getLogger(__name__)
self.logstore = init_influx_connection(self.log, {
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
})
# Create a user
uid = db_user.create(1, "test_api_compat_user")
self.assertIsNotNone(db_user.get(uid))
with db.engine.connect() as connection:
result = connection.execute(text("""
SELECT *
FROM "user"
WHERE id = :id
"""), {
"id": uid,
})
row = result.fetchone()
self.user = User(row['id'], row['created'], row['musicbrainz_id'], row['auth_token'])
def handle_new_releases_of_top_artists(message):
user_id = message["user_id"]
# need to check whether user exists before inserting otherwise possible FK error.
user = db_user.get(user_id)
if not user:
return
year_in_music.insert_new_releases_of_top_artists(user_id, message["data"])
def handle_recommendations(data):
""" Take recommended recordings for a user and save it in the db.
"""
user_id = data['user_id']
user = db_user.get(user_id)
if not user:
current_app.logger.info(f"Generated recommendations for a user that doesn't exist in the Postgres database: {user_id}")
return
current_app.logger.debug("inserting recommendation for {}".format(user["musicbrainz_id"]))
recommendations = data['recommendations']
try:
db_recommendations_cf_recording.insert_user_recommendation(
user_id,
UserRecommendationsJson(**{
'top_artist': recommendations['top_artist'],
'similar_artist': recommendations['similar_artist']
})
)
except ValidationError:
current_app.logger.error(f"""ValidationError while inserting recommendations for user with musicbrainz_id:
{user["musicbrainz_id"]}. \nData: {json.dumps(data, indent=3)}""")
current_app.logger.debug("recommendation for {} inserted".format(user["musicbrainz_id"]))
def handle_missing_musicbrainz_data(data):
""" Insert user missing musicbrainz data i.e data submitted to ListenBrainz but not MusicBrainz.
"""
user_id = data['user_id']
user = db_user.get(user_id)
if not user:
return
current_app.logger.debug(f"Inserting missing musicbrainz data for {user['musicbrainz_id']}")
missing_musicbrainz_data = data['missing_musicbrainz_data']
source = data['source']
try:
db_missing_musicbrainz_data.insert_user_missing_musicbrainz_data(
user['id'],
UserMissingMusicBrainzDataJson(missing_musicbrainz_data=missing_musicbrainz_data),
source
)
except ValidationError:
current_app.logger.error(f"""
ValidationError while inserting missing MusicBrainz data from source "{source}" for user with musicbrainz_id:
{user["musicbrainz_id"]}. Data: {json.dumps(data, indent=3)}""", exc_info=True)
current_app.logger.debug(f"Missing musicbrainz data for {user['musicbrainz_id']} inserted")
def insert_recordings(connection, playlist_id: int, recordings: List[model_playlist.WritablePlaylistRecording],
starting_position: int):
"""Insert recordings to an existing playlist. The position field will be computed based on the order
of the provided recordings.
Arguments:
connection: an open database connection
playlist_id: the playlist id to add the recordings to
recordings: a list of recordings to add
starting_position: The position number to set in the first recording. The first recording in a playlist is position 0
"""
for position, recording in enumerate(recordings, starting_position):
recording.playlist_id = playlist_id
recording.position = position
query = sqlalchemy.text("""
INSERT INTO playlist.playlist_recording (playlist_id, position, mbid, added_by_id, created)
VALUES (:playlist_id, :position, :mbid, :added_by_id, :created)
RETURNING id, created""")
return_recordings = []
user_id_map = {}
insert_ts = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
for recording in recordings:
if not recording.created:
recording.created = insert_ts
result = connection.execute(query, recording.dict(include={'playlist_id', 'position', 'mbid', 'added_by_id', 'created'}))
if recording.added_by_id not in user_id_map:
# TODO: Do this lookup in bulk
user_id_map[recording.added_by_id] = db_user.get(recording.added_by_id)
row = result.fetchone()
recording.id = row['id']
recording.created = row['created']
recording.added_by = user_id_map[recording.added_by_id]["musicbrainz_id"]
return_recordings.append(model_playlist.PlaylistRecording.parse_obj(recording.dict()))
return return_recordings
def test_update_user_details(self):
user_id = db_user.create(17, "barbazfoo", "*****@*****.**")
db_user.update_user_details(user_id, "hello-world",
"*****@*****.**")
user = db_user.get(user_id, fetch_email=True)
self.assertEqual(user["id"], user_id)
self.assertEqual(user["musicbrainz_id"], "hello-world")
self.assertEqual(user["email"], "*****@*****.**")
def setUp(self):
super().setUp()
self.app = create_app()
self.tempdir = tempfile.mkdtemp()
self.runner = CliRunner()
self.listenstore = timescale_connection._ts
self.user_id = db_user.create(1, 'iliekcomputers')
self.user_name = db_user.get(self.user_id)['musicbrainz_id']
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
if user.token_expired:
try:
user = spotify.refresh_user_token(user)
except spotify.SpotifyAPIError:
current_app.logger.error(
'Could not refresh user token from spotify', exc_info=True)
raise
else:
if user is None:
current_app.logger.debug(
"%s has revoked spotify authorization", str(user))
listenbrainz_user = db_user.get(user.user_id)
currently_playing = get_user_currently_playing(user)
if currently_playing is not None and 'item' in currently_playing:
current_app.logger.debug('Received a currently playing track for %s',
str(user))
listens = parse_and_validate_spotify_plays([currently_playing['item']],
LISTEN_TYPE_PLAYING_NOW)
submit_listens_to_listenbrainz(listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_PLAYING_NOW)
recently_played = get_user_recently_played(user)
if recently_played is not None and 'items' in recently_played:
listens = parse_and_validate_spotify_plays(recently_played['items'],
LISTEN_TYPE_IMPORT)
current_app.logger.debug('Received %d tracks for %s', len(listens),
str(user))
# if we don't have any new listens, return
if len(listens) == 0:
return
latest_listened_at = max(listen['listened_at'] for listen in listens)
submit_listens_to_listenbrainz(listenbrainz_user,
listens,
listen_type=LISTEN_TYPE_IMPORT)
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
current_app.logger.info('imported %d listens for %s' %
(len(listens), str(user)))
def get_collaborators_names_from_ids(collaborator_ids: List[int]):
collaborators = []
# TODO: Look this up in one query
for user_id in collaborator_ids:
user = db_user.get(user_id)
if user:
collaborators.append(user["musicbrainz_id"])
collaborators.sort()
return collaborators
def test_fetch_email(self):
musicbrainz_id = "one"
email = "*****@*****.**"
user_id = db_user.create(1, musicbrainz_id, email)
self.assertNotIn("email", db_user.get(user_id))
self.assertEqual(email,
db_user.get(user_id, fetch_email=True)["email"])
token = db_user.get(user_id)["auth_token"]
self.assertNotIn("email", db_user.get_by_token(token))
self.assertEqual(
email,
db_user.get_by_token(token, fetch_email=True)["email"])
self.assertNotIn("email", db_user.get_by_mb_id(musicbrainz_id))
self.assertEqual(
email,
db_user.get_by_mb_id(musicbrainz_id, fetch_email=True)["email"])
def setUp(self):
super(SpotifyDatabaseTestCase, self).setUp()
db_user.create(1, 'testspotifyuser')
self.user = db_user.get(1)
db_spotify.create_spotify(
user_id=self.user['id'],
user_token='token',
refresh_token='refresh_token',
token_expires_ts=int(time.time()),
)
def setUp(self):
super(SpotifyDatabaseTestCase, self).setUp()
db_user.create(1, 'testspotifyuser')
self.user = db_user.get(1)
db_spotify.create_spotify(
user_id=self.user['id'],
user_token='token',
refresh_token='refresh_token',
token_expires_ts=int(time.time()),
)
def test_delete(self, mock_publish_data_to_queue):
self.temporary_login(self.user['id'])
r = self.client.get(url_for('profile.delete'))
self.assert200(r)
r = self.client.post(url_for('profile.delete'), data={'token': self.user['auth_token']})
mock_publish_data_to_queue.assert_called_once()
self.assertRedirects(r, '/')
user = db_user.get(self.user['id'])
self.assertIsNone(user)
def test_delete(self):
self.temporary_login(self.user['id'])
r = self.client.get(url_for('profile.delete'))
self.assert200(r)
r = self.client.post(url_for('profile.delete'),
data={'token': self.user['auth_token']})
self.assertRedirects(r, '/')
user = db_user.get(self.user['id'])
self.assertIsNone(user)
def record_listens(request, data):
""" Submit the listen in the lastfm format to be inserted in db.
Accepts listens for both track.updateNowPlaying and track.scrobble methods.
"""
output_format = data.get('format', 'xml')
try:
sk, api_key = data['sk'], data['api_key']
except KeyError:
raise InvalidAPIUsage(CompatError.INVALID_PARAMETERS, output_format=output_format) # Invalid parameters
session = Session.load(sk)
if not session:
if not Token.is_valid_api_key(api_key):
raise InvalidAPIUsage(CompatError.INVALID_API_KEY, output_format=output_format) # Invalid API_KEY
raise InvalidAPIUsage(CompatError.INVALID_SESSION_KEY, output_format=output_format) # Invalid Session KEY
lookup = defaultdict(dict)
for key, value in data.items():
if key in ["sk", "token", "api_key", "method", "api_sig"]:
continue
matches = re.match('(.*)\[(\d+)\]', key)
if matches:
key = matches.group(1)
number = matches.group(2)
else:
number = 0
lookup[number][key] = value
if request.form['method'].lower() == 'track.updatenowplaying':
for i, listen in lookup.items():
if 'timestamp' not in listen:
listen['timestamp'] = calendar.timegm(datetime.now().utctimetuple())
# Convert to native payload then submit 'em after validation.
listen_type, native_payload = _to_native_api(lookup, data['method'], output_format)
for listen in native_payload:
validate_listen(listen, listen_type)
user = db_user.get(session.user_id)
augmented_listens = insert_payload(native_payload, user, listen_type=listen_type)
# With corrections than the original submitted listen.
doc, tag, text = Doc().tagtext()
with tag('lfm', status='ok'):
if listen_type == 'playing_now':
doc.asis(create_response_for_single_listen(list(lookup.values())[0], augmented_listens[0], listen_type))
else:
accepted_listens = len(lookup.values())
# Currently LB accepts all the listens and ignores none
with tag('scrobbles', accepted=accepted_listens, ignored='0'):
for original_listen, augmented_listen in zip(list(lookup.values()), augmented_listens):
doc.asis(create_response_for_single_listen(original_listen, augmented_listen, listen_type))
return format_response('<?xml version="1.0" encoding="utf-8"?>\n' + yattag.indent(doc.getvalue()),
output_format)
def setUp(self):
super(OAuthDatabaseTestCase, self).setUp()
db_user.create(1, 'testspotifyuser')
self.user = db_user.get(1)
db_oauth.save_token(user_id=self.user['id'],
service=ExternalServiceType.SPOTIFY,
access_token='token',
refresh_token='refresh_token',
token_expires_ts=int(time.time()),
record_listens=True,
scopes=['user-read-recently-played'])
def test_delete(self):
user_id = db_user.create(10, 'frank')
user = db_user.get(user_id)
self.assertIsNotNone(user)
db_stats.insert_user_stats(
user_id=user_id,
artists={},
recordings={},
releases={},
artist_count=2,
)
user_stats = db_stats.get_all_user_stats(user_id)
self.assertIsNotNone(user_stats)
db_user.delete(user_id)
user = db_user.get(user_id)
self.assertIsNone(user)
user_stats = db_stats.get_all_user_stats(user_id)
self.assertIsNone(user_stats)
def test_delete(self):
user_id = db_user.create(10, 'frank')
user = db_user.get(user_id)
self.assertIsNotNone(user)
with open(self.path_to_data_file('user_top_artists_db.json')) as f:
artists_data = ujson.load(f)
db_stats.insert_user_artists(
user_id=user_id,
artists=UserArtistStatJson(**{'all_time': artists_data}),
)
user_stats = db_stats.get_user_artists(user_id, 'all_time')
self.assertIsNotNone(user_stats)
db_user.delete(user_id)
user = db_user.get(user_id)
self.assertIsNone(user)
user_stats = db_stats.get_user_artists(user_id, 'all_time')
self.assertIsNone(user_stats)
def test_delete(self):
user_id = db_user.create(10, 'frank')
user = db_user.get(user_id)
self.assertIsNotNone(user)
with open(self.path_to_data_file('user_top_artists_db.json')) as f:
artists_data = ujson.load(f)
db_stats.insert_user_jsonb_data(
user_id=user_id,
stats_type='artists',
stats=StatRange[EntityRecord](**artists_data),
)
user_stats = db_stats.get_user_stats(user_id, 'all_time', 'artists')
self.assertIsNotNone(user_stats)
db_user.delete(user_id)
user = db_user.get(user_id)
self.assertIsNone(user)
user_stats = db_stats.get_user_stats(user_id, 'all_time', 'artists')
self.assertIsNone(user_stats)
def setUp(self):
super(TestAPICompatTokenClass, self).setUp()
self.log = logging.getLogger(__name__)
# Create a user
uid = db_user.create(1, "test")
self.assertIsNotNone(db_user.get(uid))
with db.engine.connect() as connection:
result = connection.execute(text('SELECT * FROM "user" WHERE id = :id'),
{"id": uid})
row = result.fetchone()
self.user = User(row['id'], row['created'], row['musicbrainz_id'], row['auth_token'])
def process_one_user(user):
""" Get recently played songs for this user and submit them to ListenBrainz.
Args:
user (spotify.Spotify): the user whose plays are to be imported.
Raises:
spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
or if we get errors while submitting the data to ListenBrainz
"""
if user.token_expired:
user = spotify.refresh_user_token(user)
listenbrainz_user = db_user.get(user.user_id)
try:
recently_played = get_user_recently_played(user)
except (spotify.SpotifyListenBrainzError, spotify.SpotifyAPIError) as e:
raise
# convert listens to ListenBrainz format and validate them
listens = []
if 'items' in recently_played:
current_app.logger.debug('Received %d listens from Spotify for %s', len(recently_played['items']), str(user))
for item in recently_played['items']:
listen = _convert_spotify_play_to_listen(item)
try:
validate_listen(listen, LISTEN_TYPE_IMPORT)
listens.append(listen)
except BadRequest:
current_app.logger.error('Could not validate listen for user %s: %s', str(user), json.dumps(listen, indent=3), exc_info=True)
# TODO: api_utils exposes werkzeug exceptions, if it's a more generic module it shouldn't be web-specific
latest_listened_at = max(listen['listened_at'] for listen in listens)
# try to submit listens to ListenBrainz
retries = 10
while retries >= 0:
try:
current_app.logger.debug('Submitting %d listens for user %s', len(listens), str(user))
insert_payload(listens, listenbrainz_user, listen_type=LISTEN_TYPE_IMPORT)
current_app.logger.debug('Submitted!')
break
except (InternalServerError, ServiceUnavailable) as e:
retries -= 1
current_app.logger.info('ISE while trying to import listens for %s: %s', str(user), str(e))
if retries == 0:
raise spotify.SpotifyListenBrainzError('ISE while trying to import listens: %s', str(e))
# we've succeeded so update the last_updated field for this user
spotify.update_latest_listened_at(user.user_id, latest_listened_at)
spotify.update_last_updated(user.user_id)
def setUp(self):
super(TestAPICompatTokenClass, self).setUp()
self.log = logging.getLogger(__name__)
# Create a user
uid = db_user.create(1, "test")
self.assertIsNotNone(db_user.get(uid))
with db.engine.connect() as connection:
result = connection.execute(
text('SELECT * FROM "user" WHERE id = :id'), {"id": uid})
row = result.fetchone()
self.user = User(row['id'], row['created'], row['musicbrainz_id'],
row['auth_token'])
def setUp(self):
super(TestInfluxListenStore, self).setUp()
self.log = logging.getLogger(__name__)
# In order to do counting correctly, we need a clean DB to start with
self.reset_influx_db()
self.logstore = init_influx_connection(self.log, {
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
})
self.testuser_id = db_user.create(1, "test")
self.testuser_name = db_user.get(self.testuser_id)['musicbrainz_id']
def submit_now_playing():
""" Handle now playing notifications sent by clients """
current_app.logger.info(json.dumps(request.form, indent=4))
try:
session = _get_session(request.form.get('s', ''))
except BadRequest:
return 'BADSESSION\n', 401
listen = _to_native_api(request.form, append_key='')
if listen is None:
return 'FAILED Invalid data submitted!\n', 400
listens = [listen]
user = db_user.get(session.user_id)
insert_payload(listens, user, LISTEN_TYPE_PLAYING_NOW)
return 'OK\n'
def load_user(user_id):
user = db_user.get(user_id)
if user:
return User.from_dbrow(user)
else:
return None