def vote(mbid, spotify_uri, user_id):
"""Submit report about incorrect Spotify mapping.
Returns:
Returns two values. First one is a boolean that indicates whether the submission has been successful.
The second is an exception in case errors occur. If there are no errors, this value is None.
"""
try:
if _base_url is None or _key is None:
raise ValueError(
"Missing MBSPOTIFY_BASE_URI or MBSPOTIFY_ACCESS_KEY.")
session = requests.Session()
session.mount(_base_url, HTTPAdapter(max_retries=2))
resp = session.post(_base_url + 'mapping/vote',
params={'key': _key},
headers={'Content-Type': 'application/json'},
data=json.dumps({
'mbid': str(mbid),
'spotify_uri': str(spotify_uri),
'user': str(user_id)
}))
cache.delete(mbid, _CACHE_NAMESPACE)
return resp.status_code == 200, None
except (RequestException, ValueError) as e:
return False, e
def delete_listen(self, listened_at: int, user_name: str,
recording_msid: str):
""" Delete a particular listen for user with specified MusicBrainz ID.
Args:
listened_at: The timestamp of the listen
user_name: the username of the user
recording_msid: the MessyBrainz ID of the recording
Raises: TimescaleListenStoreException if unable to delete the listen
"""
args = {
'listened_at': listened_at,
'user_name': user_name,
'recording_msid': recording_msid
}
query = """DELETE FROM listen
WHERE listened_at = :listened_at
AND user_name = :user_name
AND data -> 'track_metadata' -> 'additional_info' ->> 'recording_msid' = :recording_msid """
try:
with timescale.engine.connect() as connection:
connection.execute(sqlalchemy.text(query), args)
user_key = "{}{}".format(
self.ns + REDIS_TIMESCALE_USER_LISTEN_COUNT, user_name)
cache.delete(user_key)
except psycopg2.OperationalError as e:
self.log.error("Cannot delete listen for user: %s" % str(e))
raise TimescaleListenStoreException
def insert(self, listens):
""" Insert a batch of listens.
"""
submit = []
user_names = {}
for listen in listens:
user_names[listen.user_name] = 1
submit.append(listen.to_influx(quote(listen.user_name)))
if not self.influx.write_points(submit, time_precision='s'):
self.log.error(
"Cannot write data to influx. (write_points returned False), data=%s",
json.dumps(submit, indent=3))
# If we reach this point, we were able to write the listens to the InfluxListenStore.
# So update the listen counts of the users cached in brainzutils cache.
for data in submit:
user_key = "{}{}".format(REDIS_INFLUX_USER_LISTEN_COUNT,
data['fields']['user_name'])
cached_count = cache.get(user_key, decode=False)
if cached_count:
cache.increment(user_key)
# Invalidate cached data for user
for user_name in user_names.keys():
cache.delete(REDIS_USER_TIMESTAMPS % user_name)
if len(listens):
# Enter a measurement to count items inserted
submit = [{
'measurement': TEMP_COUNT_MEASUREMENT,
'tags': {
COUNT_MEASUREMENT_NAME: len(listens)
},
'fields': {
COUNT_MEASUREMENT_NAME: len(listens)
}
}]
try:
if not self.influx.write_points(submit):
self.log.error(
"Cannot write listen cound to influx. (write_points returned False)"
)
except (InfluxDBServerError, InfluxDBClientError,
ValueError) as err:
self.log.error("Cannot write data to influx: %s, data: %s",
str(err),
json.dumps(submit, indent=3),
exc_info=True)
raise
def vote(mbid, spotify_uri, user_id):
"""Submit report about incorrect Spotify mapping."""
if _base_url is None or _key is None:
return
# TODO(roman): Catch errors during voting.
requests.post(_base_url + 'mapping/vote?key=' + _key, headers={'Content-Type': 'application/json'},
data=json.dumps({
'mbid': str(mbid),
'user': str(user_id),
'spotify_uri': str(spotify_uri),
}))
cache.delete(mbid, _CACHE_NAMESPACE)
def test_delete_with_namespace(self):
key = "testing"
namespace = "spaaaaaaace"
self.assertTrue(cache.set(key, u"Пример", namespace=namespace))
self.assertEqual(cache.get(key, namespace=namespace), u"Пример")
self.assertEqual(cache.delete(key, namespace=namespace), 1)
self.assertIsNone(cache.get(key, namespace=namespace))
def test_delete_with_namespace(self):
key = "testing"
namespace = "spaaaaaaace"
self.assertTrue(cache.set(key, u"Пример", namespace=namespace))
self.assertEqual(cache.get(key, namespace=namespace), u"Пример")
self.assertEqual(cache.delete(key, namespace=namespace), 1)
self.assertIsNone(cache.get(key, namespace=namespace))
def insert(self, listens):
"""
Insert a batch of listens. Returns a list of (listened_at, track_name, user_name) that indicates
which rows were inserted into the DB. If the row is not listed in the return values, it was a duplicate.
"""
submit = []
user_names = {}
for listen in listens:
user_names[listen.user_name] = 1
submit.append(listen.to_timescale())
query = """INSERT INTO listen (listened_at, track_name, user_name, data)
VALUES %s
ON CONFLICT (listened_at, track_name, user_name)
DO NOTHING
RETURNING listened_at, track_name, user_name"""
inserted_rows = []
conn = timescale.engine.raw_connection()
with conn.cursor() as curs:
try:
execute_values(curs, query, submit, template=None)
while True:
result = curs.fetchone()
if not result:
break
inserted_rows.append((result[0], result[1], result[2]))
except UntranslatableCharacter:
conn.rollback()
return
conn.commit()
# So update the listen counts of the users cached in brainzutils cache.
for _, _, user_name in inserted_rows:
user_key = "{}{}".format(
self.ns + REDIS_TIMESCALE_USER_LISTEN_COUNT, user_name)
cached_count = cache.get(user_key, decode=False)
if cached_count:
cache.increment(user_key)
# Invalidate cached data for user
for user_name in user_names:
cache.delete(self.ns + REDIS_USER_TIMESTAMPS % user_name)
return inserted_rows
def test_get_total_listen_count(self):
total_count = self.logstore.get_total_listen_count()
self.assertEqual(total_count, 0)
count_user_1 = self._create_test_data(self.testuser["musicbrainz_id"],
self.testuser["id"])
uid = random.randint(2000, 1 << 31)
testuser2 = db_user.get_or_create(uid, f"user_{uid}")
count_user_2 = self._create_test_data(testuser2["musicbrainz_id"],
testuser2["id"])
cache.delete(REDIS_TOTAL_LISTEN_COUNT)
add_missing_to_listen_users_metadata()
update_user_listen_data()
total_count = self.logstore.get_total_listen_count()
self.assertEqual(total_count, count_user_1 + count_user_2)
def test_delete_single_listen(self):
uid = random.randint(2000, 1 << 31)
testuser = db_user.get_or_create(uid, "user_%d" % uid)
testuser_name = testuser['musicbrainz_id']
self._create_test_data(testuser_name, testuser["id"])
listens, min_ts, max_ts = self.logstore.fetch_listens(user=testuser,
to_ts=1400000300)
self.assertEqual(len(listens), 5)
self.assertEqual(listens[0].ts_since_epoch, 1400000200)
self.assertEqual(listens[1].ts_since_epoch, 1400000150)
self.assertEqual(listens[2].ts_since_epoch, 1400000100)
self.assertEqual(listens[3].ts_since_epoch, 1400000050)
self.assertEqual(listens[4].ts_since_epoch, 1400000000)
self.logstore.delete_listen(1400000050, testuser["id"],
"c7a41965-9f1e-456c-8b1d-27c0f0dde280")
pending = self._get_pending_deletes()
self.assertEqual(len(pending), 1)
self.assertEqual(pending[0]["listened_at"], 1400000050)
self.assertEqual(pending[0]["user_id"], testuser["id"])
self.assertEqual(str(pending[0]["recording_msid"]),
"c7a41965-9f1e-456c-8b1d-27c0f0dde280")
delete_listens_and_update_user_listen_data()
# clear cache entry so that count is fetched from db again
cache.delete(REDIS_USER_LISTEN_COUNT + str(testuser["id"]))
listens, min_ts, max_ts = self.logstore.fetch_listens(user=testuser,
to_ts=1400000300)
self.assertEqual(len(listens), 4)
self.assertEqual(listens[0].ts_since_epoch, 1400000200)
self.assertEqual(listens[1].ts_since_epoch, 1400000150)
self.assertEqual(listens[2].ts_since_epoch, 1400000100)
self.assertEqual(listens[3].ts_since_epoch, 1400000000)
self.assertEqual(
self.logstore.get_listen_count_for_user(testuser["id"]), 4)
min_ts, max_ts = self.logstore.get_timestamps_for_user(testuser["id"])
self.assertEqual(min_ts, 1400000000)
self.assertEqual(max_ts, 1400000200)
def insert(self, listens):
""" Insert a batch of listens.
"""
submit = []
user_names = {}
for listen in listens:
user_names[listen.user_name] = 1
submit.append(listen.to_influx(quote(listen.user_name)))
if not self.influx.write_points(submit, time_precision='s'):
self.log.error("Cannot write data to influx. (write_points returned False), data=%s", json.dumps(submit, indent=3))
# If we reach this point, we were able to write the listens to the InfluxListenStore.
# So update the listen counts of the users cached in brainzutils cache.
for data in submit:
user_key = "{}{}".format(REDIS_INFLUX_USER_LISTEN_COUNT, data['fields']['user_name'])
cached_count = cache.get(user_key, decode=False)
if cached_count:
cache.increment(user_key)
# Invalidate cached data for user
for user_name in user_names.keys():
cache.delete(REDIS_USER_TIMESTAMPS % user_name)
if len(listens):
# Enter a measurement to count items inserted
submit = [{
'measurement': TEMP_COUNT_MEASUREMENT,
'tags': {
COUNT_MEASUREMENT_NAME: len(listens)
},
'fields': {
COUNT_MEASUREMENT_NAME: len(listens)
}
}]
try:
if not self.influx.write_points(submit):
self.log.error("Cannot write listen cound to influx. (write_points returned False)")
except (InfluxDBServerError, InfluxDBClientError, ValueError) as err:
self.log.error("Cannot write data to influx: %s, data: %s", str(err), json.dumps(submit, indent=3), exc_info=True)
raise
def vote(mbid, spotify_uri, user_id):
"""Submit report about incorrect Spotify mapping.
Returns:
Returns two values. First one is a boolean that indicates whether the submission has been successful.
The second is an exception in case errors occur. If there are no errors, this value is None.
"""
try:
if _base_url is None or _key is None:
raise ValueError("Missing MBSPOTIFY_BASE_URI or MBSPOTIFY_ACCESS_KEY.")
session = requests.Session()
session.mount(_base_url, HTTPAdapter(max_retries=2))
resp = session.post(_base_url + 'mapping/vote',
params={'key': _key},
headers={'Content-Type': 'application/json'},
data=json.dumps({'mbid': str(mbid), 'spotify_uri': str(spotify_uri), 'user': str(user_id)}))
cache.delete(mbid, _CACHE_NAMESPACE)
return resp.status_code == 200, None
except (RequestException, ValueError) as e:
return False, e
def add_mapping(mbid, spotify_uri, user_id):
"""Submit new Spotify mapping.
Returns:
Returns two values. First one is a boolean that indicates whether the submission has been successful.
The second is an exception in case errors occur. If there are no errors, this value is None.
"""
if _base_url is None or _key is None:
return False, None
try:
session = requests.Session()
session.mount(_base_url, HTTPAdapter(max_retries=2))
resp = session.post(_base_url + 'mapping/add?key=' + _key,
headers={'Content-Type': 'application/json'},
data=json.dumps({'mbid': str(mbid), 'spotify_uri': spotify_uri, 'user': str(user_id)}))
cache.delete(mbid, _CACHE_NAMESPACE)
return resp.status_code == 200, None
except RequestException as e:
return False, e
def test_delete(self):
key = "testing"
self.assertTrue(cache.set(key, u"Пример"))
self.assertEqual(cache.get(key), u"Пример")
self.assertEqual(cache.delete(key), 1)
self.assertIsNone(cache.get(key))
def test_delete_listens(self):
"""
Test delete listens for a user
"""
# test get requests to delete-listens view first
self.temporary_login(self.user['login_id'])
resp = self.client.get(url_for('profile.delete_listens'))
self.assert200(resp)
# send three listens for the user
resp = self.send_listens()
self.assert200(resp)
# set the latest_import ts to a non-default value, so that we can check it was
# reset later
val = int(time.time())
resp = self.client.post(
url_for('api_v1.latest_import'),
data=json.dumps({'ts': val}),
headers={
'Authorization': 'Token {}'.format(self.user['auth_token'])
},
content_type='application/json',
)
self.assert200(resp)
resp = self.client.get(
url_for('api_v1.latest_import',
user_name=self.user['musicbrainz_id']))
self.assert200(resp)
self.assertEqual(resp.json['latest_import'], val)
self.assertNotEqual(
self.redis.ttl(
cache._prep_key(REDIS_USER_LISTEN_COUNT +
str(self.user['id']))), 0)
# check that listens have been successfully submitted
resp = self.client.get(
url_for('api_v1.get_listen_count',
user_name=self.user['musicbrainz_id']))
self.assert200(resp)
self.assertEqual(json.loads(resp.data)['payload']['count'], 3)
# now delete all the listens we just sent
# we do a get request first to put the CSRF token in the flask global context
# so that we can access it for using in the post request in the next step
self.client.get(url_for('profile.delete_listens'))
resp = self.client.post(url_for('profile.delete_listens'),
data={'csrf_token': g.csrf_token})
self.assertRedirects(
resp, url_for('user.profile',
user_name=self.user['musicbrainz_id']))
# listen counts are cached for 5 min, so delete key otherwise cached will be returned
cache.delete(REDIS_USER_LISTEN_COUNT + str(self.user['id']))
# check that listens have been successfully deleted
resp = self.client.get(
url_for('api_v1.get_listen_count',
user_name=self.user['musicbrainz_id']))
self.assert200(resp)
self.assertEqual(json.loads(resp.data)['payload']['count'], 0)
# check that the latest_import timestamp has been reset too
resp = self.client.get(
url_for('api_v1.latest_import',
user_name=self.user['musicbrainz_id']))
self.assert200(resp)
self.assertEqual(resp.json['latest_import'], 0)
def test_delete(self):
key = "testing"
self.assertTrue(cache.set(key, u"Пример"))
self.assertEqual(cache.get(key), u"Пример")
self.assertEqual(cache.delete(key), 1)
self.assertIsNone(cache.get(key))
