def test_export_feedback_streaming(self, mock_fetch_feedback): self.temporary_login(self.user['login_id']) # Three example feedback, with only basic data for the purpose of this test. feedback = [ Feedback( recording_msid='6c617681-281e-4dae-af59-8e00f93c4376', score=1, user_id=1, ), Feedback( recording_msid='7ad53fd7-5b40-4e13-b680-52716fb86d5f', score=1, user_id=1, ), Feedback( recording_msid='7816411a-2cc6-4e43-b7a1-60ad093c2c31', score=-1, user_id=1, ), ] # We expect three calls to get_feedback_for_user, and we return two, one, and # zero feedback in the batch. This tests that we fetch all batches. mock_fetch_feedback.side_effect = [feedback[0:2], feedback[2:3], []] r = self.client.post(url_for('profile.export_feedback')) self.assert200(r) # r.json returns None, so we decode the response manually. results = ujson.loads(r.data.decode('utf-8')) self.assertDictEqual(results[0], { 'recording_mbid': None, 'recording_msid': '6c617681-281e-4dae-af59-8e00f93c4376', 'score': 1, 'user_id': None, 'created': None, 'track_metadata': None, }) self.assertDictEqual(results[1], { 'recording_mbid': None, 'recording_msid': '7ad53fd7-5b40-4e13-b680-52716fb86d5f', 'score': 1, 'user_id': None, 'created': None, 'track_metadata': None, }) self.assertDictEqual(results[2], { 'recording_mbid': None, 'recording_msid': '7816411a-2cc6-4e43-b7a1-60ad093c2c31', 'score': -1, 'user_id': None, 'created': None, 'track_metadata': None, })
def get_feedback_for_multiple_recordings_for_user( user_id: int, recording_list: List[str]) -> List[Feedback]: """ Get a list of recording feedback given by the user for given recordings Args: user_id: the row ID of the user in the DB recording_list: list of recording_msid for which feedback records are to be obtained - if record is present then return it - if record is not present then return a pseudo record with score = 0 Returns: A list of Feedback objects """ args = {"user_id": user_id, "recording_list": recording_list} query = """ WITH rf AS ( SELECT user_id, recording_msid::text, score FROM recording_feedback WHERE recording_feedback.user_id=:user_id ) SELECT COALESCE(rf.user_id, :user_id) AS user_id, "user".musicbrainz_id AS user_name, rec_msid AS recording_msid, COALESCE(rf.score, 0) AS score FROM UNNEST(:recording_list) rec_msid LEFT OUTER JOIN rf ON rf.recording_msid::text = rec_msid JOIN "user" ON "user".id = :user_id """ with db.engine.connect() as connection: result = connection.execute(sqlalchemy.text(query), args) return [Feedback(**dict(row)) for row in result.fetchall()]
def test_dump_recording_feedback(self): # create a user with self.app.app_context(): one_id = db_user.create(1, 'test_user') user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # insert a feedback record feedback = Feedback( user_id=one_id, recording_msid="d23f4719-9212-49f0-ad08-ddbfbfc50d6f", score=1) db_feedback.insert(feedback) # do a db dump and reset the db private_dump, private_ts_dump, public_dump, public_ts_dump = db_dump.dump_postgres_db( self.tempdir) self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) self.assertEqual( db_feedback.get_feedback_count_for_user(user_id=one_id), 0) # import the dump and check the records are inserted db_dump.import_postgres_dump(private_dump, None, public_dump, None) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) dumped_feedback = db_feedback.get_feedback_for_user(user_id=one_id, limit=1, offset=0) self.assertEqual(len(dumped_feedback), 1) self.assertEqual(dumped_feedback[0].user_id, feedback.user_id) self.assertEqual(dumped_feedback[0].recording_msid, feedback.recording_msid) self.assertEqual(dumped_feedback[0].score, feedback.score) # reset again, and use more threads to import self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) dumped_feedback = [] db_dump.import_postgres_dump(private_dump, None, public_dump, None, threads=2) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) dumped_feedback = db_feedback.get_feedback_for_user(user_id=one_id, limit=1, offset=0) self.assertEqual(len(dumped_feedback), 1) self.assertEqual(dumped_feedback[0].user_id, feedback.user_id) self.assertEqual(dumped_feedback[0].recording_msid, feedback.recording_msid) self.assertEqual(dumped_feedback[0].score, feedback.score)
def test_update_score_when_feedback_already_exits(self): update_fb = self.sample_feedback[0] count = self.insert_test_data(self.user["id"]) result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0) self.assertEqual(len(result), count) self.assertEqual(result[1].recording_msid, update_fb["recording_msid"]) self.assertEqual(result[1].score, 1) update_fb["score"] = -1 # change the score to -1 # update a record by inserting a record with updated score value db_feedback.insert( Feedback(user_id=self.user["id"], recording_msid=update_fb["recording_msid"], score=update_fb["score"])) result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0) self.assertEqual(len(result), count) self.assertEqual(result[0].recording_msid, update_fb["recording_msid"]) self.assertEqual(result[0].score, -1)
def get_feedback_for_user(user_id: int, limit: int, offset: int, score: int = None): """ Get a list of recording feedback given by the user in descending order of their creation Args: user_id: the row ID of the user in the DB score: the score value by which the results are to be filtered. If 1 then returns the loved recordings, if -1 returns hated recordings. limit: number of rows to be returned offset: number of feedback to skip from the beginning Returns: A list of Feedback objects """ args = {"user_id": user_id, "limit": limit, "offset": offset} query = """ SELECT user_id, "user".musicbrainz_id AS user_name, recording_msid::text, score FROM recording_feedback JOIN "user" ON "user".id = recording_feedback.user_id WHERE user_id = :user_id """ if score: query += "AND score = :score" args["score"] = score query += """ ORDER BY recording_feedback.created DESC LIMIT :limit OFFSET :offset """ with db.engine.connect() as connection: result = connection.execute(sqlalchemy.text(query), args) return [Feedback(**dict(row)) for row in result.fetchall()]
def test_delete(self): del_fb = self.sample_feedback[0] count = self.insert_test_data(self.user["id"]) result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0) self.assertEqual(len(result), count) self.assertEqual(result[3].recording_msid, del_fb["recording_msid"]) # delete one record for the user using msid db_feedback.delete( Feedback(user_id=self.user["id"], recording_msid=del_fb["recording_msid"], score=del_fb["score"])) result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0) self.assertEqual(len(result), 3) self.assertNotIn(del_fb["recording_msid"], [x.recording_msid for x in result]) # delete using mbid db_feedback.delete( Feedback(user_id=self.user["id"], recording_mbid=self.sample_feedback[2]["recording_mbid"], score=self.sample_feedback[2]["score"])) result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0) self.assertEqual(len(result), 2) self.assertNotIn(self.sample_feedback[2]["recording_mbid"], [x.recording_mbid for x in result]) # delete using mbid and msid both db_feedback.delete( Feedback(user_id=self.user["id"], recording_mbid=self.sample_feedback[3]["recording_mbid"], recording_msid=self.sample_feedback[3]["recording_msid"], score=self.sample_feedback[2]["score"])) result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0) self.assertEqual(len(result), 1) self.assertNotIn(self.sample_feedback[3]["recording_mbid"], [x.recording_mbid for x in result])
def insert_test_data(self, user_id, neg_score=False): """ Insert test data into the database """ for fb in self.sample_feedback: db_feedback.insert( Feedback(user_id=user_id, recording_msid=fb["recording_msid"], score=fb["score"])) return len(self.sample_feedback)
def test_create_feedback(self, mock_notify): self.user = db_user.get_or_create(1, "ernie") self.user2 = db_user.get_or_create(2, "bert") sample_feedback = [{ "user_id": self.user['id'], "recording_msid": "d23f4719-9212-49f0-ad08-ddbfbfc50d6f", "score": 1 }, { "user_id": self.user2['id'], "recording_msid": "222eb00d-9ead-42de-aec9-8f8c1509413d", "score": -1 }] for fb in sample_feedback: db_feedback.insert( Feedback(user_id=fb["user_id"], recording_msid=fb["recording_msid"], score=fb["score"])) rec_feedback = [{ "recording_mbid": "d23f4719-9212-49f0-ad08-ddbfbfc50d6f", "rating": 'love', 'user_id': self.user['id'] }, { "recording_mbid": "222eb00d-9ead-42de-aec9-8f8c1509413d", "rating": 'bad_recommendation', "user_id": self.user['id'] }, { "recording_mbid": "922eb00d-9ead-42de-aec9-8f8c1509413d", "rating": 'hate', "user_id": self.user2['id'] }] for fb in rec_feedback: db_rec_feedback.insert( RecommendationFeedbackSubmit( user_id=fb['user_id'], recording_mbid=fb["recording_mbid"], rating=fb["rating"])) # create a feedback dump self.runner.invoke(dump_manager.create_feedback, ['--location', self.tempdir]) self.assertEqual(len(os.listdir(self.tempdir)), 1) dump_name = os.listdir(self.tempdir)[0] mock_notify.assert_called_with(dump_name, 'feedback') # make sure that the dump contains a feedback dump archive_count = 0 for file_name in os.listdir(os.path.join(self.tempdir, dump_name)): if file_name.endswith('.tar.xz'): archive_count += 1 self.assertEqual(archive_count, 1)
def insert_test_data(self, user_id): sample_feedback = [{ "recording_msid": "d23f4719-9212-49f0-ad08-ddbfbfc50d6f", "score": 1 }, { "recording_msid": "222eb00d-9ead-42de-aec9-8f8c1509413d", "score": -1 }] for fb in sample_feedback: db_feedback.insert( Feedback(user_id=user_id, recording_msid=fb["recording_msid"], score=fb["score"])) return sample_feedback
def recording_feedback(): """ Submit recording feedback (love/hate) to the server. A user token (found on https://listenbrainz.org/profile/ ) must be provided in the Authorization header! Each request should contain only one feedback in the payload. For complete details on the format of the JSON to be POSTed to this endpoint, see :ref:`feedback-json-doc`. :reqheader Authorization: Token <user token> :statuscode 200: feedback accepted. :statuscode 400: invalid JSON sent, see error message for details. :statuscode 401: invalid authorization. See error message for details. :resheader Content-Type: *application/json* """ user = _validate_auth_header() data = request.json if 'recording_msid' not in data or 'score' not in data: log_raise_400( "JSON document must contain recording_msid and " "score top level keys", data) if 'recording_msid' in data and 'score' in data and len(data) > 2: log_raise_400( "JSON document may only contain recording_msid and " "score top level keys", data) try: feedback = Feedback(user_id=user["id"], recording_msid=data["recording_msid"], score=data["score"]) except ValidationError as e: # Validation errors from the Pydantic model are multi-line. While passing it as a response the new lines # are displayed as \n. str.replace() to tidy up the error message so that it becomes a good one line error message. log_raise_400( "Invalid JSON document submitted: %s" % str(e).replace("\n ", ":").replace("\n", " "), data) try: if feedback.score == 0: db_feedback.delete(feedback) else: db_feedback.insert(feedback) except Exception as e: current_app.logger.error( "Error while inserting recording feedback: {}".format(e)) raise APIInternalServerError("Something went wrong. Please try again.") return jsonify({'status': 'ok'})
def insert_test_data_with_metadata(self, user_id, neg_score=False): """ Insert test data with metadata into the database """ with msb_db.engine.connect() as connection: msid = get_id_from_meta_hash(connection, self.sample_recording) if msid is None: msid = submit_recording(connection, self.sample_recording) msid = str(msid) artists = load_recordings_from_msids(connection, [msid]) self.saved_artist_msid = artists[0]["ids"]["artist_msid"] self.sample_feedback_with_metadata[0]["recording_msid"] = msid query = """INSERT INTO mbid_mapping_metadata (recording_mbid, release_mbid, release_name, artist_credit_id, artist_mbids, artist_credit_name, recording_name) VALUES ('076255b4-1575-11ec-ac84-135bf6a670e3', '1fd178b4-1575-11ec-b98a-d72392cd8c97', 'release_name', 65, '{6a221fda-2200-11ec-ac7d-dfa16a57158f}'::UUID[], 'artist name', 'recording name')""" with ts.engine.connect() as connection: connection.execute(sqlalchemy.text(query)) query = """INSERT INTO mbid_mapping (recording_msid, recording_mbid, match_type, last_updated) VALUES (:msid, :mbid, :match_type, now())""" with ts.engine.connect() as connection: connection.execute( sqlalchemy.text(query), { "msid": msid, "mbid": "076255b4-1575-11ec-ac84-135bf6a670e3", "match_type": "exact_match" }) for fb in self.sample_feedback_with_metadata: db_feedback.insert( Feedback(user_id=user_id, recording_msid=fb["recording_msid"], score=fb["score"])) return len(self.sample_feedback_with_metadata)
def recording_feedback(): """ Submit recording feedback (love/hate) to the server. A user token (found on https://listenbrainz.org/profile/ ) must be provided in the Authorization header! Each request should contain only one feedback in the payload. For complete details on the format of the JSON to be POSTed to this endpoint, see :ref:`feedback-json-doc`. :reqheader Authorization: Token <user token> :statuscode 200: feedback accepted. :statuscode 400: invalid JSON sent, see error message for details. :statuscode 401: invalid authorization. See error message for details. :resheader Content-Type: *application/json* """ user = validate_auth_header() data = request.json if ('recording_msid' not in data and 'recording_mbid' not in data) or 'score' not in data: log_raise_400( "JSON document must contain either recording_msid or recording_mbid, and " "score top level keys", data) if set(data) - {"recording_msid", "recording_mbid", "score"}: log_raise_400( "JSON document may only contain recording_msid, recording_mbid and " "score top level keys", data) try: feedback = Feedback(user_id=user["id"], recording_msid=data.get("recording_msid", None), recording_mbid=data.get("recording_mbid", None), score=data["score"]) except ValidationError as e: # Validation errors from the Pydantic model are multi-line. While passing it as a response the new lines # are displayed as \n. str.replace() to tidy up the error message so that it becomes a good one line error message. log_raise_400( "Invalid JSON document submitted: %s" % str(e).replace("\n ", ":").replace("\n", " "), data) if feedback.score == FEEDBACK_DEFAULT_SCORE: db_feedback.delete(feedback) else: db_feedback.insert(feedback) return jsonify({'status': 'ok'})
def test_delete(self): del_fb = self.sample_feedback[0] count = self.insert_test_data(self.user["id"]) result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0) self.assertEqual(len(result), count) self.assertEqual(result[1].recording_msid, del_fb["recording_msid"]) # delete one record for the user db_feedback.delete( Feedback( user_id=self.user["id"], recording_msid=del_fb["recording_msid"], score=del_fb["score"] ) ) result = db_feedback.get_feedback_for_user(user_id=self.user["id"], limit=25, offset=0) self.assertEqual(len(result), 1) self.assertNotEqual(result[0].recording_msid, del_fb["recording_msid"])
def get_feedback_for_recording(recording_type: str, recording: str, limit: int, offset: int, score: int = None)\ -> List[Feedback]: """ Get a list of recording feedback for a given recording in descending order of their creation Args: recording_type: type of id, recording_msid or recording_mbid recording: the msid or mbid of the recording score: the score value by which the results are to be filtered. If 1 then returns the loved recordings, if -1 returns hated recordings. limit: number of rows to be returned offset: number of feedback to skip from the beginning Returns: A list of Feedback objects """ args = {"recording": recording, "limit": limit, "offset": offset} query = """ SELECT user_id , "user".musicbrainz_id AS user_name , recording_msid::text , recording_mbid::text , score , recording_feedback.created FROM recording_feedback JOIN "user" ON "user".id = recording_feedback.user_id WHERE """ + recording_type + " = :recording" if score: query += " AND score = :score" args["score"] = score query += """ ORDER BY recording_feedback.created DESC LIMIT :limit OFFSET :offset """ with db.engine.connect() as connection: result = connection.execute(text(query), args) return [Feedback(**dict(row)) for row in result.fetchall()]
def insert_test_data_with_metadata(self, user_id, neg_score=False): """ Insert test data with metadata into the database """ with msb_db.engine.connect() as connection: msid = submit_recording(connection, self.sample_recording) artists = load_recordings_from_msids(connection, [msid]) self.saved_artist_msid = artists[0]["ids"]["artist_msid"] self.sample_feedback_with_metadata[0]["recording_msid"] = msid query = """INSERT INTO listen_mbid_mapping (id, recording_mbid, release_mbid, artist_credit_id, artist_mbids, artist_credit_name, recording_name, match_type) VALUES (1, '076255b4-1575-11ec-ac84-135bf6a670e3', '1fd178b4-1575-11ec-b98a-d72392cd8c97', 65, '{6a221fda-2200-11ec-ac7d-dfa16a57158f}'::UUID[], 'artist name', 'recording name', 'exact_match')""" with ts.engine.connect() as connection: connection.execute(sqlalchemy.text(query)) query = """INSERT INTO listen_join_listen_mbid_mapping (recording_msid, listen_mbid_mapping) VALUES ('%s', 1)""" % msid with ts.engine.connect() as connection: connection.execute(sqlalchemy.text(query)) for fb in self.sample_feedback_with_metadata: db_feedback.insert( Feedback(user_id=user_id, recording_msid=fb["recording_msid"], score=fb["score"])) return len(self.sample_feedback_with_metadata)
def get_feedback_for_multiple_recordings_for_user( user_id: int, user_name: str, recording_msids: List[str], recording_mbids: List[str]) -> List[Feedback]: """ Get a list of recording feedback given by the user for given recordings For each recording msid and recording mbid, - if record is present then return it - if record is not present then return a pseudo record with score = 0 Args: user_id: the row ID of the user in the DB user_name: the user name of the user, not used in the query but only for creating the response to be returned from the api recording_msids: list of recording_msid for which feedback records are to be obtained recording_mbids: list of recording_mbid for which feedback records are to be obtained Returns: A list of Feedback objects """ params = {"user_id": user_id} query_base = """ WITH rf AS ( SELECT user_id, recording_msid::text, recording_mbid::text, score FROM recording_feedback WHERE recording_feedback.user_id = :user_id ) """ query_msid = """ SELECT recording_msid , recording_mbid , COALESCE(rf.score, 0) AS score FROM UNNEST(:recording_msids) recording_msid LEFT OUTER JOIN rf USING (recording_msid) """ query_mbid = """ SELECT recording_msid , recording_mbid , COALESCE(rf.score, 0) AS score FROM UNNEST(:recording_mbids) recording_mbid LEFT OUTER JOIN rf USING (recording_mbid) """ # we cannot use single query here because recordings parameter passed to UNNEST should # not be empty so that we check which list is not empty and construct the query accordingly if recording_msids and recording_mbids: # both msid and mbid list are not empty params["recording_msids"] = recording_msids params["recording_mbids"] = recording_mbids query_remaining = query_msid + " UNION " + query_mbid elif recording_msids: # only msid list is not empty params["recording_msids"] = recording_msids query_remaining = query_msid else: # only mbid list is not empty params["recording_mbids"] = recording_mbids query_remaining = query_mbid query = query_base + query_remaining with db.engine.connect() as connection: result = connection.execute(text(query), params) return [ Feedback(user_id=user_id, user_name=user_name, **dict(row)) for row in result.fetchall() ]
def get_feedback_for_user(user_id: int, limit: int, offset: int, score: int = None, metadata: bool = False) -> List[Feedback]: """ Get a list of recording feedback given by the user in descending order of their creation Args: user_id: the row ID of the user in the DB score: the score value by which the results are to be filtered. If 1 then returns the loved recordings, if -1 returns hated recordings. limit: number of rows to be returned offset: number of feedback to skip from the beginning metadata: fetch metadata for the returned feedback recordings Returns: A list of Feedback objects """ args = {"user_id": user_id, "limit": limit, "offset": offset} query = """ SELECT user_id, "user".musicbrainz_id AS user_name, recording_msid::text, score, recording_feedback.created FROM recording_feedback JOIN "user" ON "user".id = recording_feedback.user_id WHERE user_id = :user_id """ if score: query += " AND score = :score" args["score"] = score query += """ ORDER BY recording_feedback.created DESC LIMIT :limit OFFSET :offset """ with db.engine.connect() as connection: result = connection.execute(sqlalchemy.text(query), args) feedback = [Feedback(**dict(row)) for row in result.fetchall()] if metadata and len(feedback) > 0: msids = [f.recording_msid for f in feedback] index = {f.recording_msid: f for f in feedback} # Fetch the artist and track names from MSB with msb_db.engine.connect() as connection: try: msb_recordings = load_recordings_from_msids(connection, msids) except NoDataFoundException: msb_recordings = [] artist_msids = {} if msb_recordings: for rec in msb_recordings: index[rec["ids"]["recording_msid"]].track_metadata = { "artist_name": rec["payload"]["artist"], "release_name": rec["payload"].get("release_name", ""), "track_name": rec["payload"]["title"] } artist_msids[rec["ids"] ["recording_msid"]] = rec["ids"]["artist_msid"] # Fetch the mapped MBIDs from the mapping query = """SELECT recording_msid::TEXT, m.recording_mbid::TEXT, release_mbid::TEXT, artist_mbids::TEXT[] FROM mbid_mapping m JOIN mbid_mapping_metadata mm ON m.recording_mbid = mm.recording_mbid WHERE recording_msid in :msids ORDER BY recording_msid""" with timescale.engine.connect() as connection: result = connection.execute(sqlalchemy.text(query), msids=tuple(msids)) for row in result.fetchall(): if row["recording_mbid"] is not None: index[row["recording_msid"]].track_metadata[ 'additional_info'] = { "recording_mbid": row["recording_mbid"], "release_mbid": row["release_mbid"], "artist_mbids": row["artist_mbids"], "artist_msid": artist_msids[row["recording_msid"]] } return feedback
def _feedback_to_api(fb: Feedback) -> dict: fb.user_id = fb.user_name del fb.user_name return fb.dict()