def test_export_feedback_streaming(self, mock_fetch_feedback):
self.temporary_login(self.user['login_id'])
# Three example feedback, with only basic data for the purpose of this test.
feedback = [
Feedback(
recording_msid='6c617681-281e-4dae-af59-8e00f93c4376',
score=1,
user_id=1,
),
Feedback(
recording_msid='7ad53fd7-5b40-4e13-b680-52716fb86d5f',
score=1,
user_id=1,
),
Feedback(
recording_msid='7816411a-2cc6-4e43-b7a1-60ad093c2c31',
score=-1,
user_id=1,
),
]
# We expect three calls to get_feedback_for_user, and we return two, one, and
# zero feedback in the batch. This tests that we fetch all batches.
mock_fetch_feedback.side_effect = [feedback[0:2], feedback[2:3], []]
r = self.client.post(url_for('profile.export_feedback'))
self.assert200(r)
# r.json returns None, so we decode the response manually.
results = ujson.loads(r.data.decode('utf-8'))
self.assertDictEqual(results[0], {
'recording_mbid': None,
'recording_msid': '6c617681-281e-4dae-af59-8e00f93c4376',
'score': 1,
'user_id': None,
'created': None,
'track_metadata': None,
})
self.assertDictEqual(results[1], {
'recording_mbid': None,
'recording_msid': '7ad53fd7-5b40-4e13-b680-52716fb86d5f',
'score': 1,
'user_id': None,
'created': None,
'track_metadata': None,
})
self.assertDictEqual(results[2], {
'recording_mbid': None,
'recording_msid': '7816411a-2cc6-4e43-b7a1-60ad093c2c31',
'score': -1,
'user_id': None,
'created': None,
'track_metadata': None,
})
def get_feedback_for_multiple_recordings_for_user(
user_id: int, recording_list: List[str]) -> List[Feedback]:
""" Get a list of recording feedback given by the user for given recordings
Args:
user_id: the row ID of the user in the DB
recording_list: list of recording_msid for which feedback records are to be obtained
- if record is present then return it
- if record is not present then return a pseudo record with score = 0
Returns:
A list of Feedback objects
"""
args = {"user_id": user_id, "recording_list": recording_list}
query = """ WITH rf AS (
SELECT user_id, recording_msid::text, score
FROM recording_feedback
WHERE recording_feedback.user_id=:user_id
)
SELECT COALESCE(rf.user_id, :user_id) AS user_id,
"user".musicbrainz_id AS user_name,
rec_msid AS recording_msid,
COALESCE(rf.score, 0) AS score
FROM UNNEST(:recording_list) rec_msid
LEFT OUTER JOIN rf
ON rf.recording_msid::text = rec_msid
JOIN "user"
ON "user".id = :user_id """
with db.engine.connect() as connection:
result = connection.execute(sqlalchemy.text(query), args)
return [Feedback(**dict(row)) for row in result.fetchall()]
def test_dump_recording_feedback(self):
# create a user
with self.app.app_context():
one_id = db_user.create(1, 'test_user')
user_count = db_user.get_user_count()
self.assertEqual(user_count, 1)
# insert a feedback record
feedback = Feedback(
user_id=one_id,
recording_msid="d23f4719-9212-49f0-ad08-ddbfbfc50d6f",
score=1)
db_feedback.insert(feedback)
# do a db dump and reset the db
private_dump, private_ts_dump, public_dump, public_ts_dump = db_dump.dump_postgres_db(
self.tempdir)
self.reset_db()
user_count = db_user.get_user_count()
self.assertEqual(user_count, 0)
self.assertEqual(
db_feedback.get_feedback_count_for_user(user_id=one_id), 0)
# import the dump and check the records are inserted
db_dump.import_postgres_dump(private_dump, None, public_dump, None)
user_count = db_user.get_user_count()
self.assertEqual(user_count, 1)
dumped_feedback = db_feedback.get_feedback_for_user(user_id=one_id,
limit=1,
offset=0)
self.assertEqual(len(dumped_feedback), 1)
self.assertEqual(dumped_feedback[0].user_id, feedback.user_id)
self.assertEqual(dumped_feedback[0].recording_msid,
feedback.recording_msid)
self.assertEqual(dumped_feedback[0].score, feedback.score)
# reset again, and use more threads to import
self.reset_db()
user_count = db_user.get_user_count()
self.assertEqual(user_count, 0)
dumped_feedback = []
db_dump.import_postgres_dump(private_dump,
None,
public_dump,
None,
threads=2)
user_count = db_user.get_user_count()
self.assertEqual(user_count, 1)
dumped_feedback = db_feedback.get_feedback_for_user(user_id=one_id,
limit=1,
offset=0)
self.assertEqual(len(dumped_feedback), 1)
self.assertEqual(dumped_feedback[0].user_id, feedback.user_id)
self.assertEqual(dumped_feedback[0].recording_msid,
feedback.recording_msid)
self.assertEqual(dumped_feedback[0].score, feedback.score)
def test_update_score_when_feedback_already_exits(self):
update_fb = self.sample_feedback[0]
count = self.insert_test_data(self.user["id"])
result = db_feedback.get_feedback_for_user(user_id=self.user["id"],
limit=25,
offset=0)
self.assertEqual(len(result), count)
self.assertEqual(result[1].recording_msid, update_fb["recording_msid"])
self.assertEqual(result[1].score, 1)
update_fb["score"] = -1 # change the score to -1
# update a record by inserting a record with updated score value
db_feedback.insert(
Feedback(user_id=self.user["id"],
recording_msid=update_fb["recording_msid"],
score=update_fb["score"]))
result = db_feedback.get_feedback_for_user(user_id=self.user["id"],
limit=25,
offset=0)
self.assertEqual(len(result), count)
self.assertEqual(result[0].recording_msid, update_fb["recording_msid"])
self.assertEqual(result[0].score, -1)
def get_feedback_for_user(user_id: int,
limit: int,
offset: int,
score: int = None):
""" Get a list of recording feedback given by the user in descending order of their creation
Args:
user_id: the row ID of the user in the DB
score: the score value by which the results are to be filtered. If 1 then returns the loved recordings,
if -1 returns hated recordings.
limit: number of rows to be returned
offset: number of feedback to skip from the beginning
Returns:
A list of Feedback objects
"""
args = {"user_id": user_id, "limit": limit, "offset": offset}
query = """ SELECT user_id, "user".musicbrainz_id AS user_name, recording_msid::text, score
FROM recording_feedback
JOIN "user"
ON "user".id = recording_feedback.user_id
WHERE user_id = :user_id """
if score:
query += "AND score = :score"
args["score"] = score
query += """ ORDER BY recording_feedback.created DESC
LIMIT :limit OFFSET :offset """
with db.engine.connect() as connection:
result = connection.execute(sqlalchemy.text(query), args)
return [Feedback(**dict(row)) for row in result.fetchall()]
def test_delete(self):
del_fb = self.sample_feedback[0]
count = self.insert_test_data(self.user["id"])
result = db_feedback.get_feedback_for_user(user_id=self.user["id"],
limit=25,
offset=0)
self.assertEqual(len(result), count)
self.assertEqual(result[3].recording_msid, del_fb["recording_msid"])
# delete one record for the user using msid
db_feedback.delete(
Feedback(user_id=self.user["id"],
recording_msid=del_fb["recording_msid"],
score=del_fb["score"]))
result = db_feedback.get_feedback_for_user(user_id=self.user["id"],
limit=25,
offset=0)
self.assertEqual(len(result), 3)
self.assertNotIn(del_fb["recording_msid"],
[x.recording_msid for x in result])
# delete using mbid
db_feedback.delete(
Feedback(user_id=self.user["id"],
recording_mbid=self.sample_feedback[2]["recording_mbid"],
score=self.sample_feedback[2]["score"]))
result = db_feedback.get_feedback_for_user(user_id=self.user["id"],
limit=25,
offset=0)
self.assertEqual(len(result), 2)
self.assertNotIn(self.sample_feedback[2]["recording_mbid"],
[x.recording_mbid for x in result])
# delete using mbid and msid both
db_feedback.delete(
Feedback(user_id=self.user["id"],
recording_mbid=self.sample_feedback[3]["recording_mbid"],
recording_msid=self.sample_feedback[3]["recording_msid"],
score=self.sample_feedback[2]["score"]))
result = db_feedback.get_feedback_for_user(user_id=self.user["id"],
limit=25,
offset=0)
self.assertEqual(len(result), 1)
self.assertNotIn(self.sample_feedback[3]["recording_mbid"],
[x.recording_mbid for x in result])
def insert_test_data(self, user_id, neg_score=False):
""" Insert test data into the database """
for fb in self.sample_feedback:
db_feedback.insert(
Feedback(user_id=user_id,
recording_msid=fb["recording_msid"],
score=fb["score"]))
return len(self.sample_feedback)
def test_create_feedback(self, mock_notify):
self.user = db_user.get_or_create(1, "ernie")
self.user2 = db_user.get_or_create(2, "bert")
sample_feedback = [{
"user_id": self.user['id'],
"recording_msid": "d23f4719-9212-49f0-ad08-ddbfbfc50d6f",
"score": 1
}, {
"user_id": self.user2['id'],
"recording_msid": "222eb00d-9ead-42de-aec9-8f8c1509413d",
"score": -1
}]
for fb in sample_feedback:
db_feedback.insert(
Feedback(user_id=fb["user_id"],
recording_msid=fb["recording_msid"],
score=fb["score"]))
rec_feedback = [{
"recording_mbid": "d23f4719-9212-49f0-ad08-ddbfbfc50d6f",
"rating": 'love',
'user_id': self.user['id']
}, {
"recording_mbid": "222eb00d-9ead-42de-aec9-8f8c1509413d",
"rating": 'bad_recommendation',
"user_id": self.user['id']
}, {
"recording_mbid": "922eb00d-9ead-42de-aec9-8f8c1509413d",
"rating": 'hate',
"user_id": self.user2['id']
}]
for fb in rec_feedback:
db_rec_feedback.insert(
RecommendationFeedbackSubmit(
user_id=fb['user_id'],
recording_mbid=fb["recording_mbid"],
rating=fb["rating"]))
# create a feedback dump
self.runner.invoke(dump_manager.create_feedback,
['--location', self.tempdir])
self.assertEqual(len(os.listdir(self.tempdir)), 1)
dump_name = os.listdir(self.tempdir)[0]
mock_notify.assert_called_with(dump_name, 'feedback')
# make sure that the dump contains a feedback dump
archive_count = 0
for file_name in os.listdir(os.path.join(self.tempdir, dump_name)):
if file_name.endswith('.tar.xz'):
archive_count += 1
self.assertEqual(archive_count, 1)
def insert_test_data(self, user_id):
sample_feedback = [{
"recording_msid": "d23f4719-9212-49f0-ad08-ddbfbfc50d6f",
"score": 1
}, {
"recording_msid": "222eb00d-9ead-42de-aec9-8f8c1509413d",
"score": -1
}]
for fb in sample_feedback:
db_feedback.insert(
Feedback(user_id=user_id,
recording_msid=fb["recording_msid"],
score=fb["score"]))
return sample_feedback
def recording_feedback():
"""
Submit recording feedback (love/hate) to the server. A user token (found on https://listenbrainz.org/profile/ )
must be provided in the Authorization header! Each request should contain only one feedback in the payload.
For complete details on the format of the JSON to be POSTed to this endpoint, see :ref:`feedback-json-doc`.
:reqheader Authorization: Token <user token>
:statuscode 200: feedback accepted.
:statuscode 400: invalid JSON sent, see error message for details.
:statuscode 401: invalid authorization. See error message for details.
:resheader Content-Type: *application/json*
"""
user = _validate_auth_header()
data = request.json
if 'recording_msid' not in data or 'score' not in data:
log_raise_400(
"JSON document must contain recording_msid and "
"score top level keys", data)
if 'recording_msid' in data and 'score' in data and len(data) > 2:
log_raise_400(
"JSON document may only contain recording_msid and "
"score top level keys", data)
try:
feedback = Feedback(user_id=user["id"],
recording_msid=data["recording_msid"],
score=data["score"])
except ValidationError as e:
# Validation errors from the Pydantic model are multi-line. While passing it as a response the new lines
# are displayed as \n. str.replace() to tidy up the error message so that it becomes a good one line error message.
log_raise_400(
"Invalid JSON document submitted: %s" %
str(e).replace("\n ", ":").replace("\n", " "), data)
try:
if feedback.score == 0:
db_feedback.delete(feedback)
else:
db_feedback.insert(feedback)
except Exception as e:
current_app.logger.error(
"Error while inserting recording feedback: {}".format(e))
raise APIInternalServerError("Something went wrong. Please try again.")
return jsonify({'status': 'ok'})
def insert_test_data_with_metadata(self, user_id, neg_score=False):
""" Insert test data with metadata into the database """
with msb_db.engine.connect() as connection:
msid = get_id_from_meta_hash(connection, self.sample_recording)
if msid is None:
msid = submit_recording(connection, self.sample_recording)
msid = str(msid)
artists = load_recordings_from_msids(connection, [msid])
self.saved_artist_msid = artists[0]["ids"]["artist_msid"]
self.sample_feedback_with_metadata[0]["recording_msid"] = msid
query = """INSERT INTO mbid_mapping_metadata
(recording_mbid, release_mbid, release_name, artist_credit_id,
artist_mbids, artist_credit_name, recording_name)
VALUES ('076255b4-1575-11ec-ac84-135bf6a670e3',
'1fd178b4-1575-11ec-b98a-d72392cd8c97',
'release_name',
65,
'{6a221fda-2200-11ec-ac7d-dfa16a57158f}'::UUID[],
'artist name', 'recording name')"""
with ts.engine.connect() as connection:
connection.execute(sqlalchemy.text(query))
query = """INSERT INTO mbid_mapping
(recording_msid, recording_mbid, match_type, last_updated)
VALUES (:msid, :mbid, :match_type, now())"""
with ts.engine.connect() as connection:
connection.execute(
sqlalchemy.text(query), {
"msid": msid,
"mbid": "076255b4-1575-11ec-ac84-135bf6a670e3",
"match_type": "exact_match"
})
for fb in self.sample_feedback_with_metadata:
db_feedback.insert(
Feedback(user_id=user_id,
recording_msid=fb["recording_msid"],
score=fb["score"]))
return len(self.sample_feedback_with_metadata)
def recording_feedback():
"""
Submit recording feedback (love/hate) to the server. A user token (found on https://listenbrainz.org/profile/ )
must be provided in the Authorization header! Each request should contain only one feedback in the payload.
For complete details on the format of the JSON to be POSTed to this endpoint, see :ref:`feedback-json-doc`.
:reqheader Authorization: Token <user token>
:statuscode 200: feedback accepted.
:statuscode 400: invalid JSON sent, see error message for details.
:statuscode 401: invalid authorization. See error message for details.
:resheader Content-Type: *application/json*
"""
user = validate_auth_header()
data = request.json
if ('recording_msid' not in data
and 'recording_mbid' not in data) or 'score' not in data:
log_raise_400(
"JSON document must contain either recording_msid or recording_mbid, and "
"score top level keys", data)
if set(data) - {"recording_msid", "recording_mbid", "score"}:
log_raise_400(
"JSON document may only contain recording_msid, recording_mbid and "
"score top level keys", data)
try:
feedback = Feedback(user_id=user["id"],
recording_msid=data.get("recording_msid", None),
recording_mbid=data.get("recording_mbid", None),
score=data["score"])
except ValidationError as e:
# Validation errors from the Pydantic model are multi-line. While passing it as a response the new lines
# are displayed as \n. str.replace() to tidy up the error message so that it becomes a good one line error message.
log_raise_400(
"Invalid JSON document submitted: %s" %
str(e).replace("\n ", ":").replace("\n", " "), data)
if feedback.score == FEEDBACK_DEFAULT_SCORE:
db_feedback.delete(feedback)
else:
db_feedback.insert(feedback)
return jsonify({'status': 'ok'})
def test_delete(self):
del_fb = self.sample_feedback[0]
count = self.insert_test_data(self.user["id"])
result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0)
self.assertEqual(len(result), count)
self.assertEqual(result[1].recording_msid, del_fb["recording_msid"])
# delete one record for the user
db_feedback.delete(
Feedback(
user_id=self.user["id"],
recording_msid=del_fb["recording_msid"],
score=del_fb["score"]
)
)
result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0)
self.assertEqual(len(result), 1)
self.assertNotEqual(result[0].recording_msid, del_fb["recording_msid"])
def get_feedback_for_recording(recording_type: str, recording: str, limit: int, offset: int, score: int = None)\
-> List[Feedback]:
""" Get a list of recording feedback for a given recording in descending order of their creation
Args:
recording_type: type of id, recording_msid or recording_mbid
recording: the msid or mbid of the recording
score: the score value by which the results are to be filtered. If 1 then returns the loved recordings,
if -1 returns hated recordings.
limit: number of rows to be returned
offset: number of feedback to skip from the beginning
Returns:
A list of Feedback objects
"""
args = {"recording": recording, "limit": limit, "offset": offset}
query = """
SELECT user_id
, "user".musicbrainz_id AS user_name
, recording_msid::text
, recording_mbid::text
, score
, recording_feedback.created
FROM recording_feedback
JOIN "user"
ON "user".id = recording_feedback.user_id
WHERE """ + recording_type + " = :recording"
if score:
query += " AND score = :score"
args["score"] = score
query += """ ORDER BY recording_feedback.created DESC
LIMIT :limit OFFSET :offset """
with db.engine.connect() as connection:
result = connection.execute(text(query), args)
return [Feedback(**dict(row)) for row in result.fetchall()]
def insert_test_data_with_metadata(self, user_id, neg_score=False):
""" Insert test data with metadata into the database """
with msb_db.engine.connect() as connection:
msid = submit_recording(connection, self.sample_recording)
artists = load_recordings_from_msids(connection, [msid])
self.saved_artist_msid = artists[0]["ids"]["artist_msid"]
self.sample_feedback_with_metadata[0]["recording_msid"] = msid
query = """INSERT INTO listen_mbid_mapping
(id, recording_mbid, release_mbid, artist_credit_id, artist_mbids,
artist_credit_name, recording_name, match_type)
VALUES (1,
'076255b4-1575-11ec-ac84-135bf6a670e3',
'1fd178b4-1575-11ec-b98a-d72392cd8c97',
65,
'{6a221fda-2200-11ec-ac7d-dfa16a57158f}'::UUID[],
'artist name', 'recording name', 'exact_match')"""
with ts.engine.connect() as connection:
connection.execute(sqlalchemy.text(query))
query = """INSERT INTO listen_join_listen_mbid_mapping
(recording_msid, listen_mbid_mapping)
VALUES ('%s', 1)""" % msid
with ts.engine.connect() as connection:
connection.execute(sqlalchemy.text(query))
for fb in self.sample_feedback_with_metadata:
db_feedback.insert(
Feedback(user_id=user_id,
recording_msid=fb["recording_msid"],
score=fb["score"]))
return len(self.sample_feedback_with_metadata)
def get_feedback_for_multiple_recordings_for_user(
user_id: int, user_name: str, recording_msids: List[str],
recording_mbids: List[str]) -> List[Feedback]:
""" Get a list of recording feedback given by the user for given recordings
For each recording msid and recording mbid,
- if record is present then return it
- if record is not present then return a pseudo record with score = 0
Args:
user_id: the row ID of the user in the DB
user_name: the user name of the user, not used in the query but only for creating the
response to be returned from the api
recording_msids: list of recording_msid for which feedback records are to be obtained
recording_mbids: list of recording_mbid for which feedback records are to be obtained
Returns:
A list of Feedback objects
"""
params = {"user_id": user_id}
query_base = """
WITH rf AS (
SELECT user_id, recording_msid::text, recording_mbid::text, score
FROM recording_feedback
WHERE recording_feedback.user_id = :user_id
)
"""
query_msid = """
SELECT recording_msid
, recording_mbid
, COALESCE(rf.score, 0) AS score
FROM UNNEST(:recording_msids) recording_msid
LEFT OUTER JOIN rf
USING (recording_msid)
"""
query_mbid = """
SELECT recording_msid
, recording_mbid
, COALESCE(rf.score, 0) AS score
FROM UNNEST(:recording_mbids) recording_mbid
LEFT OUTER JOIN rf
USING (recording_mbid)
"""
# we cannot use single query here because recordings parameter passed to UNNEST should
# not be empty so that we check which list is not empty and construct the query accordingly
if recording_msids and recording_mbids: # both msid and mbid list are not empty
params["recording_msids"] = recording_msids
params["recording_mbids"] = recording_mbids
query_remaining = query_msid + " UNION " + query_mbid
elif recording_msids: # only msid list is not empty
params["recording_msids"] = recording_msids
query_remaining = query_msid
else: # only mbid list is not empty
params["recording_mbids"] = recording_mbids
query_remaining = query_mbid
query = query_base + query_remaining
with db.engine.connect() as connection:
result = connection.execute(text(query), params)
return [
Feedback(user_id=user_id, user_name=user_name, **dict(row))
for row in result.fetchall()
]
def get_feedback_for_user(user_id: int,
limit: int,
offset: int,
score: int = None,
metadata: bool = False) -> List[Feedback]:
""" Get a list of recording feedback given by the user in descending order of their creation
Args:
user_id: the row ID of the user in the DB
score: the score value by which the results are to be filtered. If 1 then returns the loved recordings,
if -1 returns hated recordings.
limit: number of rows to be returned
offset: number of feedback to skip from the beginning
metadata: fetch metadata for the returned feedback recordings
Returns:
A list of Feedback objects
"""
args = {"user_id": user_id, "limit": limit, "offset": offset}
query = """ SELECT user_id,
"user".musicbrainz_id AS user_name,
recording_msid::text, score,
recording_feedback.created
FROM recording_feedback
JOIN "user"
ON "user".id = recording_feedback.user_id
WHERE user_id = :user_id """
if score:
query += " AND score = :score"
args["score"] = score
query += """ ORDER BY recording_feedback.created DESC
LIMIT :limit OFFSET :offset """
with db.engine.connect() as connection:
result = connection.execute(sqlalchemy.text(query), args)
feedback = [Feedback(**dict(row)) for row in result.fetchall()]
if metadata and len(feedback) > 0:
msids = [f.recording_msid for f in feedback]
index = {f.recording_msid: f for f in feedback}
# Fetch the artist and track names from MSB
with msb_db.engine.connect() as connection:
try:
msb_recordings = load_recordings_from_msids(connection, msids)
except NoDataFoundException:
msb_recordings = []
artist_msids = {}
if msb_recordings:
for rec in msb_recordings:
index[rec["ids"]["recording_msid"]].track_metadata = {
"artist_name": rec["payload"]["artist"],
"release_name": rec["payload"].get("release_name", ""),
"track_name": rec["payload"]["title"]
}
artist_msids[rec["ids"]
["recording_msid"]] = rec["ids"]["artist_msid"]
# Fetch the mapped MBIDs from the mapping
query = """SELECT recording_msid::TEXT, m.recording_mbid::TEXT, release_mbid::TEXT, artist_mbids::TEXT[]
FROM mbid_mapping m
JOIN mbid_mapping_metadata mm
ON m.recording_mbid = mm.recording_mbid
WHERE recording_msid in :msids
ORDER BY recording_msid"""
with timescale.engine.connect() as connection:
result = connection.execute(sqlalchemy.text(query),
msids=tuple(msids))
for row in result.fetchall():
if row["recording_mbid"] is not None:
index[row["recording_msid"]].track_metadata[
'additional_info'] = {
"recording_mbid": row["recording_mbid"],
"release_mbid": row["release_mbid"],
"artist_mbids": row["artist_mbids"],
"artist_msid": artist_msids[row["recording_msid"]]
}
return feedback
def _feedback_to_api(fb: Feedback) -> dict:
fb.user_id = fb.user_name
del fb.user_name
return fb.dict()