def browse_releases(artist_id=None,
release_group=None,
release_types=None,
limit=None,
offset=None,
includes=None):
"""Get all the releases by a certain artist and/or a release group.
You need to provide an artist's MusicBrainz ID or the Release Group's MusicBrainz ID
"""
if release_types is None:
release_types = []
key = cache.gen_key(artist_id, release_group, limit, offset,
*release_types, *includes)
releases = cache.get(key)
if not releases:
try:
api_resp = musicbrainzngs.browse_releases(
artist=artist_id,
release_type=release_types,
limit=limit,
offset=offset,
release_group=release_group,
includes=includes)
releases = api_resp.get('release-list')
except ResponseError as e:
if e.cause.code == 404:
return None
else:
raise InternalServerError(e.cause.msg)
cache.set(key=key, val=releases, time=DEFAULT_CACHE_EXPIRATION)
return releases
def test_single_dict_fancy(self):
dictionary = {
"fancy": u"Да",
"тест": 11,
}
cache.set('some_dict', dictionary)
self.assertEqual(cache.get('some_dict'), dictionary)
def get_total_listen_count(self, cache_value=True):
""" Returns the total number of listens stored in the ListenStore.
First checks the brainzutils cache for the value, if not present there
makes a query to the db and caches it in brainzutils cache.
"""
if cache_value:
count = cache.get(
InfluxListenStore.REDIS_INFLUX_TOTAL_LISTEN_COUNT,
decode=False)
if count:
return int(count)
try:
result = self.influx.query(
"""SELECT %s
FROM "%s"
ORDER BY time DESC
LIMIT 1""" %
(COUNT_MEASUREMENT_NAME, TIMELINE_COUNT_MEASUREMENT))
except (InfluxDBServerError, InfluxDBClientError) as err:
self.log.error("Cannot query influx: %s" % str(err), exc_info=True)
raise
try:
item = result.get_points(
measurement=TIMELINE_COUNT_MEASUREMENT).__next__()
count = int(item[COUNT_MEASUREMENT_NAME])
timestamp = convert_to_unix_timestamp(item['time'])
except (KeyError, ValueError, StopIteration):
timestamp = 0
count = 0
# Now sum counts that have been added in the interval we're interested in
try:
result = self.influx.query(
"""SELECT sum(%s) as total
FROM "%s"
WHERE time > %s""" %
(COUNT_MEASUREMENT_NAME, TEMP_COUNT_MEASUREMENT,
get_influx_query_timestamp(timestamp)))
except (InfluxDBServerError, InfluxDBClientError) as err:
self.log.error("Cannot query influx: %s" % str(err), exc_info=True)
raise
try:
data = result.get_points(
measurement=TEMP_COUNT_MEASUREMENT).__next__()
count += int(data['total'])
except StopIteration:
pass
if cache_value:
cache.set(
InfluxListenStore.REDIS_INFLUX_TOTAL_LISTEN_COUNT,
int(count),
InfluxListenStore.TOTAL_LISTEN_COUNT_CACHE_TIME,
encode=False,
)
return count
def get_total_listen_count(self, cache_value=True):
""" Returns the total number of listens stored in the ListenStore.
First checks the brainzutils cache for the value, if not present there
makes a query to the db and caches it in brainzutils cache.
"""
if cache_value:
count = cache.get(
self.ns +
TimescaleListenStore.REDIS_TIMESCALE_TOTAL_LISTEN_COUNT,
decode=False)
if count:
return int(count)
query = "SELECT SUM(count) AS value FROM listen_count"
try:
with timescale.engine.connect() as connection:
result = connection.execute(sqlalchemy.text(query))
count = int(result.fetchone()["value"] or "0")
except psycopg2.OperationalError as e:
self.log.error("Cannot query timescale listen_count: %s" % str(e),
exc_info=True)
raise
if cache_value:
cache.set(
self.ns +
TimescaleListenStore.REDIS_TIMESCALE_TOTAL_LISTEN_COUNT,
count,
TimescaleListenStore.TOTAL_LISTEN_COUNT_CACHE_TIME,
encode=False,
)
return count
def browse_release_groups(*,
artist_id,
release_types=None,
limit=None,
offset=None):
"""Get all release groups linked to an artist.
Args:
artist_id (uuid): MBID of the artist.
release_types (list): List of types of release groups to be fetched.
limit (int): Max number of release groups to return.
offset (int): Offset that can be used in conjunction with the limit.
Returns:
Tuple containing the list of dictionaries of release groups ordered by release year
and the total count of the release groups.
"""
artist_id = str(artist_id)
if release_types is None:
release_types = []
release_types = [
release_type.capitalize() for release_type in release_types
]
key = cache.gen_key(artist_id, limit, offset, *release_types)
release_groups = cache.get(key)
if not release_groups:
release_groups = db.get_release_groups_for_artist(
artist_id=artist_id,
release_types=release_types,
limit=limit,
offset=offset)
cache.set(key=key, val=release_groups, time=DEFAULT_CACHE_EXPIRATION)
return release_groups
def get_last_submitted_recordings():
"""Get list of last submitted recordings.
Returns:
List of dictionaries with basic info about last submitted recordings:
mbid (MusicBrainz ID), artist (name), and title.
"""
cache_key = "last-submitted-recordings"
last_submissions = cache.get(cache_key)
if not last_submissions:
with db.engine.connect() as connection:
# We are getting results with of offset of 10 rows because we'd
# prefer to show recordings for which we already calculated
# high-level data. This might not be the best way to do that.
result = connection.execute("""SELECT ll.gid,
llj.data->'metadata'->'tags'->'artist'->>0,
llj.data->'metadata'->'tags'->'title'->>0
FROM lowlevel ll
JOIN lowlevel_json llj
ON ll.id = llj.id
ORDER BY ll.id DESC
LIMIT 5
OFFSET 10""")
last_submissions = result.fetchall()
last_submissions = [
{
"mbid": str(r[0]),
"artist": r[1],
"title": r[2],
} for r in last_submissions if r[1] and r[2]
]
cache.set(cache_key, last_submissions, expirein=LAST_MBIDS_CACHE_TIMEOUT)
return last_submissions
def browse_release_groups(artist_id=None,
release_types=None,
limit=None,
offset=None):
"""Get all release groups linked to an artist.
You need to provide artist's MusicBrainz ID.
"""
if release_types is None:
release_types = []
key = cache.gen_key(artist_id, limit, offset, *release_types)
release_groups = cache.get(key)
if not release_groups:
try:
api_resp = musicbrainzngs.browse_release_groups(
artist=artist_id,
release_type=release_types,
limit=limit,
offset=offset)
release_groups = api_resp.get('release-group-count'), api_resp.get(
'release-group-list')
except ResponseError as e:
if e.cause.code == 404:
return None
else:
raise InternalServerError(e.cause.msg)
cache.set(key=key, val=release_groups, time=DEFAULT_CACHE_EXPIRATION)
return release_groups
def browse_release_groups(*, artist_id, release_types=None, limit=None, offset=None):
"""Get all release groups linked to an artist.
Args:
artist_id (uuid): MBID of the artist.
release_types (list): List of types of release groups to be fetched.
limit (int): Max number of release groups to return.
offset (int): Offset that can be used in conjunction with the limit.
Returns:
Tuple containing the list of dictionaries of release groups ordered by release year
and the total count of the release groups.
"""
artist_id = str(artist_id)
includes_data = defaultdict(dict)
if release_types is None:
release_types = []
release_types = [release_type.capitalize() for release_type in release_types]
key = cache.gen_key(artist_id, limit, offset, *release_types)
release_groups = cache.get(key)
if not release_groups:
with mb_session() as db:
release_groups_query = _browse_release_groups_query(db, artist_id, release_types)
count = release_groups_query.count()
release_groups = release_groups_query.order_by(
case([(models.ReleaseGroupMeta.first_release_date_year.is_(None), 1)], else_=0),
models.ReleaseGroupMeta.first_release_date_year.desc()
).limit(limit).offset(offset).all()
for release_group in release_groups:
includes_data[release_group.id]['meta'] = release_group.meta
release_groups = ([to_dict_release_groups(release_group, includes_data[release_group.id])
for release_group in release_groups], count)
cache.set(key=key, val=release_groups, time=DEFAULT_CACHE_EXPIRATION)
return release_groups
def browse_release_groups(*, artist_id, release_types=None, limit=None, offset=None):
"""Get all release groups linked to an artist.
Args:
artist_id (uuid): MBID of the artist.
release_types (list): List of types of release groups to be fetched.
limit (int): Max number of release groups to return.
offset (int): Offset that can be used in conjunction with the limit.
Returns:
Tuple containing the list of dictionaries of release groups ordered by release year
and the total count of the release groups.
"""
artist_id = str(artist_id)
includes_data = defaultdict(dict)
if release_types is None:
release_types = []
release_types = [release_type.capitalize() for release_type in release_types]
key = cache.gen_key(artist_id, limit, offset, *release_types)
release_groups = cache.get(key)
if not release_groups:
with mb_session() as db:
release_groups_query = _browse_release_groups_query(db, artist_id, release_types)
count = release_groups_query.count()
release_groups = release_groups_query.order_by(
case([(models.ReleaseGroupMeta.first_release_date_year.is_(None), 1)], else_=0),
models.ReleaseGroupMeta.first_release_date_year.desc()
).limit(limit).offset(offset).all()
for release_group in release_groups:
includes_data[release_group.id]['meta'] = release_group.meta
release_groups = ([to_dict_release_groups(release_group, includes_data[release_group.id])
for release_group in release_groups], count)
cache.set(key=key, val=release_groups, time=DEFAULT_CACHE_EXPIRATION)
return release_groups
def test_single_dict_fancy(self):
dictionary = {
"fancy": u"Да",
"тест": 11,
}
cache.set('some_dict', dictionary)
self.assertEqual(cache.get('some_dict'), dictionary)
def _fetch_access_token(refresh=False) -> str:
"""Get an access token from the OAuth credentials.
https://developer.spotify.com/web-api/authorization-guide/#client-credentials-flow
"""
key = cache.gen_key("spotify_oauth_access_token")
access_token = cache.get(key)
if refresh or not access_token:
client_id = app.config.get("SPOTIFY_CLIENT_ID")
client_secret = app.config.get("SPOTIFY_CLIENT_SECRET")
auth_value = b64encode(bytes(f"{client_id}:{client_secret}",
"utf-8")).decode("utf-8")
response = requests.post(
"https://accounts.spotify.com/api/token",
data={
"grant_type": "client_credentials"
},
headers={
"Authorization": f"Basic {auth_value}"
},
).json()
access_token = response.get("access_token")
if not access_token:
raise SpotifyException(
"Could not fetch access token for Spotify API")
# Making the token stored in cache expire at the same time as the actual token
cache.set(key=key,
val=access_token,
time=response.get("expires_in", 10))
return access_token
def get_last_submitted_recordings():
"""Get list of last submitted recordings.
Returns:
List of dictionaries with basic info about last submitted recordings:
mbid (MusicBrainz ID), artist (name), and title.
"""
cache_key = "last-submitted-recordings"
last_submissions = cache.get(cache_key)
if not last_submissions:
with db.engine.connect() as connection:
# We are getting results with of offset of 10 rows because we'd
# prefer to show recordings for which we already calculated
# high-level data. This might not be the best way to do that.
result = connection.execute("""SELECT ll.gid,
llj.data->'metadata'->'tags'->'artist'->>0,
llj.data->'metadata'->'tags'->'title'->>0
FROM lowlevel ll
JOIN lowlevel_json llj
ON ll.id = llj.id
ORDER BY ll.id DESC
LIMIT 5
OFFSET 10""")
last_submissions = result.fetchall()
last_submissions = [
{
"mbid": str(r[0]),
"artist": r[1],
"title": r[2],
} for r in last_submissions if r[1] and r[2]
]
cache.set(cache_key, last_submissions, time=LAST_MBIDS_CACHE_TIMEOUT)
return last_submissions
def mappings(mbid=None):
"""Get mappings to Spotify for a specified MusicBrainz ID.
Returns:
List containing Spotify URIs that are mapped to specified MBID.
"""
if _base_url is None:
flash.warn(lazy_gettext(_UNAVAILABLE_MSG))
return []
data = cache.get(mbid, _CACHE_NAMESPACE)
if not data:
try:
session = requests.Session()
session.mount(_base_url, HTTPAdapter(max_retries=2))
resp = session.post(
url=_base_url + 'mapping',
headers={'Content-Type': 'application/json'},
data=json.dumps({'mbid': mbid}),
)
resp.raise_for_status()
data = resp.json().get('mappings')
except RequestException:
flash.warn(lazy_gettext("Spotify mapping server is unavailable. You will not see an embedded player."))
return []
cache.set(key=mbid, namespace=_CACHE_NAMESPACE, val=data)
return data
def reset_listen_count(self, user_name):
""" Reset the listen count of a user from cache and put in a new calculated value.
returns the re-calculated listen count.
Args:
user_name: the musicbrainz id of user whose listen count needs to be reset
"""
query = "SELECT SUM(count) FROM listen_count_30day WHERE user_name = :user_name"
t0 = time.monotonic()
try:
with timescale.engine.connect() as connection:
result = connection.execute(sqlalchemy.text(query), {
"user_name": user_name,
})
count = int(result.fetchone()[0] or 0)
except psycopg2.OperationalError as e:
self.log.error("Cannot query timescale listen_count: %s" % str(e),
exc_info=True)
raise
# intended for production monitoring
self.log.info("listen counts %s %.2fs" %
(user_name, time.monotonic() - t0))
# put this value into brainzutils cache without an expiry time
cache.set(REDIS_USER_LISTEN_COUNT + user_name,
count,
expirein=0,
encode=False)
return count
def mappings(mbid=None):
"""Get mappings to Spotify for a specified MusicBrainz ID.
Returns:
List containing Spotify URIs that are mapped to specified MBID.
"""
if _base_url is None:
flash.warn(lazy_gettext(_UNAVAILABLE_MSG))
return []
data = cache.get(mbid, _CACHE_NAMESPACE)
if not data:
try:
session = requests.Session()
session.mount(_base_url, HTTPAdapter(max_retries=2))
resp = session.post(
url=_base_url + 'mapping',
headers={'Content-Type': 'application/json'},
data=json.dumps({'mbid': mbid}),
)
resp.raise_for_status()
data = resp.json().get('mappings')
except RequestException:
flash.warn(
lazy_gettext(
"Spotify mapping server is unavailable. You will not see an embedded player."
))
return []
cache.set(key=mbid,
namespace=_CACHE_NAMESPACE,
val=data,
time=MBSPOTIFY_CACHE_TIMEOUT)
return data
def _get_user_count():
""" Gets user count from either the redis cache or from the database.
If not present in the cache, it makes a query to the db and stores the
result in the cache for 10 minutes.
"""
#redis_connection = _redis.redis
user_count_key = "{}.{}".format(STATS_PREFIX, 'user_count')
#if redis_connection.exists(user_count_key):
# return redis_connection.get(user_count_key)
#else:
# try:
# user_count = db_user.get_user_count()
# except DatabaseException as e:
# raise
# redis_connection.setex(user_count_key, user_count, CACHE_TIME)
# return user_count
try:
return cache.get(user_count_key)
except:
try:
user_count = db_user.get_user_count()
except DatabaseException as e:
raise
cache.set(user_count_key, user_count, CACHE_TIME)
return user_count
def get_release_group_by_id(mbid):
"""Get release group using the MusicBrainz ID."""
key = cache.gen_key(mbid)
release_group = cache.get(key)
if not release_group:
release_group = _get_release_group_by_id(mbid)
cache.set(key=key, val=release_group, time=DEFAULT_CACHE_EXPIRATION)
return release_group_rel.process(release_group)
def get_release_group_by_id(mbid):
"""Get release group using the MusicBrainz ID."""
key = cache.gen_key(mbid)
release_group = cache.get(key)
if not release_group:
release_group = _get_release_group_by_id(mbid)
cache.set(key=key, val=release_group, time=DEFAULT_CACHE_EXPIRATION)
return release_group_rel.process(release_group)
def test_key_expire(self, mock_redis):
cache.init(host='host', port=2, namespace=self.namespace)
cache.set('key', u'value'.encode('utf-8'), time=30)
expected_key = 'NS_TEST:a62f2225bf70bfaccbc7f1ef2a397836717377de'.encode('utf-8')
# msgpack encoded value
expected_value = b'\xc4\x05value'
mock_redis.return_value.mset.assert_called_with({expected_key: expected_value})
mock_redis.return_value.pexpire.assert_called_with(expected_key, 30000)
def add_stats_to_cache():
"""Compute the most recent statistics and add them to cache"""
now = datetime.datetime.now(pytz.utc)
with db.engine.connect() as connection:
stats = _count_submissions_to_date(connection, now)
cache.set(STATS_CACHE_KEY, stats,
time=STATS_CACHE_TIMEOUT, namespace=STATS_CACHE_NAMESPACE)
cache.set(STATS_CACHE_LAST_UPDATE_KEY, now,
time=STATS_CACHE_TIMEOUT, namespace=STATS_CACHE_NAMESPACE)
def put_playing_now(self, user_id, listen, expire_time):
""" Save a listen as `playing_now` for a particular time in Redis.
Args:
user_id (int): the row ID of the user
listen (dict): the listen data
expire_time (int): the time in seconds in which the `playing_now` listen should expire
"""
cache.set(self.PLAYING_NOW_KEY + str(user_id), ujson.dumps(listen).encode('utf-8'), time=expire_time)
def add_stats_to_cache():
"""Compute the most recent statistics and add them to cache"""
now = datetime.datetime.now(pytz.utc)
with db.engine.connect() as connection:
stats = _count_submissions_to_date(connection, now)
cache.set(STATS_CACHE_KEY, stats,
expirein=STATS_CACHE_TIMEOUT, namespace=STATS_CACHE_NAMESPACE)
cache.set(STATS_CACHE_LAST_UPDATE_KEY, now,
expirein=STATS_CACHE_TIMEOUT, namespace=STATS_CACHE_NAMESPACE)
def _create_test_data(self):
self.log.info("Inserting test data...")
self.listen = generate_data(self.testuser_id, MIN_ID + 1, 1)[0]
listen = self.listen.to_json()
#self._redis.redis.setex('playing_now' + ':' + str(listen['user_id']),
cache.set('playing_now' + ':' + str(listen['user_id']),
ujson.dumps(listen).encode('utf-8'),
self.config.PLAYING_NOW_MAX_DURATION)
self.log.info("Test data inserted")
def set_empty_cache_values_for_user(self, user_name):
"""When a user is created, set the listen_count and timestamp keys so that we
can avoid the expensive lookup for a brand new user."""
cache.set(REDIS_USER_LISTEN_COUNT + user_name,
0,
expirein=0,
encode=False)
cache.set(REDIS_USER_TIMESTAMPS + user_name, "0,0", expirein=0)
def test_set_key_unicode(self, mock_redis):
"""Test setting a unicode value"""
cache.init(host='host', port=2, namespace=self.namespace)
cache.set('key', u'value')
expected_key = 'NS_TEST:a62f2225bf70bfaccbc7f1ef2a397836717377de'.encode('utf-8')
# msgpack encoded value
expected_value = b'\xa5value'
mock_redis.return_value.mset.assert_called_with({expected_key: expected_value})
mock_redis.return_value.pexpire.assert_not_called()
def test_datetime(self):
self.assertTrue(cache.set('some_time', datetime.datetime.now()))
self.assertEqual(type(cache.get('some_time')), datetime.datetime)
dictionary = {
"id": 1,
"created": datetime.datetime.now(),
}
self.assertTrue(cache.set('some_other_time', dictionary))
self.assertEqual(cache.get('some_other_time'), dictionary)
def test_datetime(self):
self.assertTrue(cache.set('some_time', datetime.datetime.now()))
self.assertEqual(type(cache.get('some_time')), datetime.datetime)
dictionary = {
"id": 1,
"created": datetime.datetime.now(),
}
self.assertTrue(cache.set('some_other_time', dictionary))
self.assertEqual(cache.get('some_other_time'), dictionary)
def test_key_expire(self, mock_redis):
cache.init(host='host', port=2, namespace=self.namespace)
cache.set('key', u'value'.encode('utf-8'), time=30)
expected_key = 'NS_TEST:a62f2225bf70bfaccbc7f1ef2a397836717377de'.encode(
'utf-8')
# msgpack encoded value
expected_value = b'\xc4\x05value'
mock_redis.return_value.mset.assert_called_with(
{expected_key: expected_value})
mock_redis.return_value.pexpire.assert_called_with(expected_key, 30000)
def get_recording_by_id(mbid):
mbid = str(mbid)
recording = cache.get(mbid)
if not recording:
try:
recording = musicbrainzngs.get_recording_by_id(mbid, includes=['artists', 'releases', 'media'])['recording']
except ResponseError as e:
raise DataUnavailable(e)
cache.set(mbid, recording, time=CACHE_TIMEOUT)
return recording
def get_release_group_by_id(mbid):
"""Get release group using the MusicBrainz ID."""
key = cache.gen_key('release-group', mbid)
release_group = cache.get(key)
if not release_group:
release_group = db.fetch_multiple_release_groups(
[mbid],
includes=['artists', 'releases', 'release-group-rels', 'url-rels', 'tags'],
unknown_entities_for_missing=True,
)[mbid]
cache.set(key=key, val=release_group, time=DEFAULT_CACHE_EXPIRATION)
return release_group_rel.process(release_group)
def test_set_key_unicode(self, mock_redis):
"""Test setting a unicode value"""
cache.init(host='host', port=2, namespace=self.namespace)
cache.set('key', u'value')
expected_key = 'NS_TEST:a62f2225bf70bfaccbc7f1ef2a397836717377de'.encode(
'utf-8')
# msgpack encoded value
expected_value = b'\xa5value'
mock_redis.return_value.mset.assert_called_with(
{expected_key: expected_value})
mock_redis.return_value.pexpire.assert_not_called()
def search(query: str, *, item_types="", limit=20, offset=0) -> dict:
"""Search for items (artists, albums, or tracks) by a query string.
More information is available at https://developer.spotify.com/web-api/search-item/.
"""
cache_key = cache.gen_key(query, item_types, limit, offset)
cache_namespace = "spotify_search"
result = cache.get(cache_key, cache_namespace)
if not result:
query = urllib.parse.quote(query.encode('utf8'))
result = _get(f"search?q={query}&type={item_types}&limit={str(limit)}&offset={str(offset)}")
cache.set(key=cache_key, namespace=cache_namespace, val=result, time=_DEFAULT_CACHE_EXPIRATION)
return result
def get_total_listen_count(self, cache_value=True):
""" Returns the total number of listens stored in the ListenStore.
First checks the brainzutils cache for the value, if not present there
makes a query to the db and caches it in brainzutils cache.
"""
if cache_value:
count = cache.get(InfluxListenStore.REDIS_INFLUX_TOTAL_LISTEN_COUNT, decode=False)
if count:
return int(count)
try:
result = self.influx.query("""SELECT %s
FROM "%s"
ORDER BY time DESC
LIMIT 1""" % (COUNT_MEASUREMENT_NAME, TIMELINE_COUNT_MEASUREMENT))
except (InfluxDBServerError, InfluxDBClientError) as err:
self.log.error("Cannot query influx: %s" % str(err), exc_info=True)
raise
try:
item = result.get_points(measurement=TIMELINE_COUNT_MEASUREMENT).__next__()
count = int(item[COUNT_MEASUREMENT_NAME])
timestamp = convert_to_unix_timestamp(item['time'])
except (KeyError, ValueError, StopIteration):
timestamp = 0
count = 0
# Now sum counts that have been added in the interval we're interested in
try:
result = self.influx.query("""SELECT sum(%s) as total
FROM "%s"
WHERE time > %s""" % (COUNT_MEASUREMENT_NAME, TEMP_COUNT_MEASUREMENT, get_influx_query_timestamp(timestamp)))
except (InfluxDBServerError, InfluxDBClientError) as err:
self.log.error("Cannot query influx: %s" % str(err), exc_info=True)
raise
try:
data = result.get_points(measurement=TEMP_COUNT_MEASUREMENT).__next__()
count += int(data['total'])
except StopIteration:
pass
if cache_value:
cache.set(
InfluxListenStore.REDIS_INFLUX_TOTAL_LISTEN_COUNT,
int(count),
InfluxListenStore.TOTAL_LISTEN_COUNT_CACHE_TIME,
encode=False,
)
return count
def get_event_by_id(mbid):
"""Get event with the MusicBrainz ID.
Args:
mbid (uuid): MBID(gid) of the event.
Returns:
Dictionary containing the event information.
"""
key = cache.gen_key(mbid)
event = cache.get(key)
if not event:
event = _get_event_by_id(mbid)
cache.set(key=key, val=event, time=DEFAULT_CACHE_EXPIRATION)
return event
def get_place_by_id(mbid):
"""Get place with the MusicBrainz ID.
Args:
mbid (uuid): MBID(gid) of the place.
Returns:
Dictionary containing the place information.
"""
key = cache.gen_key(mbid)
place = cache.get(key)
if not place:
place = _get_place_by_id(mbid)
cache.set(key=key, val=place, time=DEFAULT_CACHE_EXPIRATION)
return place_rel.process(place)
def get_release_by_id(mbid):
"""Get release with the MusicBrainz ID.
Args:
mbid (uuid): MBID(gid) of the release.
Returns:
Dictionary containing the release information.
"""
key = cache.gen_key(mbid)
release = cache.get(key)
if not release:
release = _get_release_by_id(mbid)
cache.set(key=key, val=release, time=DEFAULT_CACHE_EXPIRATION)
return release
def get_release_by_id(mbid):
"""Get release with the MusicBrainz ID.
Args:
mbid (uuid): MBID(gid) of the release.
Returns:
Dictionary containing the release information.
"""
key = cache.gen_key(mbid)
release = cache.get(key)
if not release:
release = _get_release_by_id(mbid)
cache.set(key=key, val=release, time=DEFAULT_CACHE_EXPIRATION)
return release
def get_artist_by_id(mbid):
"""Get artist with MusicBrainz ID.
Args:
mbid (uuid): MBID(gid) of the artist.
Returns:
Dictionary containing the artist information
"""
key = cache.gen_key(mbid)
artist = cache.get(key)
if not artist:
artist = _get_artist_by_id(mbid)
cache.set(key=key, val=artist, time=DEFAULT_CACHE_EXPIRATION)
return artist_rel.process(artist)
def test_delete_with_namespace(self):
key = "testing"
namespace = "spaaaaaaace"
self.assertTrue(cache.set(key, u"Пример", namespace=namespace))
self.assertEqual(cache.get(key, namespace=namespace), u"Пример")
self.assertEqual(cache.delete(key, namespace=namespace), 1)
self.assertIsNone(cache.get(key, namespace=namespace))
def test_delete_with_namespace(self):
key = "testing"
namespace = "spaaaaaaace"
self.assertTrue(cache.set(key, u"Пример", namespace=namespace))
self.assertEqual(cache.get(key, namespace=namespace), u"Пример")
self.assertEqual(cache.delete(key, namespace=namespace), 1)
self.assertIsNone(cache.get(key, namespace=namespace))
def get_album(spotify_id):
"""Get Spotify catalog information for a single album.
Returns:
Album object from Spotify. More info about this type of object is
available at https://developer.spotify.com/web-api/object-model/#album-object.
"""
namespace = "spotify_album"
album = cache.get(spotify_id, namespace)
if not album:
album = requests.get("%s/albums/%s" % (BASE_URL, spotify_id)).json()
cache.set(key=spotify_id,
namespace=namespace,
val=album,
time=DEFAULT_CACHE_EXPIRATION)
return album
def _get_user_count():
""" Gets user count from either the brainzutils cache or from the database.
If not present in the cache, it makes a query to the db and stores the
result in the cache for 10 minutes.
"""
user_count_key = "{}.{}".format(STATS_PREFIX, 'user_count')
user_count = cache.get(user_count_key, decode=False)
if user_count:
return user_count
else:
try:
user_count = db_user.get_user_count()
except DatabaseException as e:
raise
cache.set(user_count_key, int(user_count), CACHE_TIME, encode=False)
return user_count
def test_single_dict(self):
dictionary = {
"fancy": "yeah",
"wow": 11,
}
self.assertTrue(cache.set('some_dict', dictionary))
self.assertEqual(cache.get('some_dict'), dictionary)
def get_label_by_id(mbid):
"""Get label with MusicBrainz ID.
Args:
mbid (uuid): MBID(gid) of the label.
Returns:
Dictionary containing the label information
"""
key = cache.gen_key('label', mbid)
label = cache.get(key)
if not label:
label = db.get_label_by_id(mbid,
includes=['artist-rels', 'url-rels'],
unknown_entities_for_missing=True)
cache.set(key=key, val=label, time=DEFAULT_CACHE_EXPIRATION)
return label_rel.process(label)
def create_record(cls, access_token, ip_address):
"""Creates new access log record with a current timestamp.
It also checks if `DIFFERENT_IP_LIMIT` is exceeded within current time
and `CLEANUP_RANGE_MINUTES`, alerts admins if that's the case.
Args:
access_token: Access token used to access the API.
ip_address: IP access used to access the API.
Returns:
New access log record.
"""
new_record = cls(
token=access_token,
ip_address=ip_address,
)
db.session.add(new_record)
db.session.commit()
# Checking if HOURLY_ALERT_THRESHOLD is exceeded
count = cls.query \
.distinct(cls.ip_address) \
.filter(cls.timestamp > datetime.now(pytz.utc) - timedelta(minutes=CLEANUP_RANGE_MINUTES),
cls.token == access_token) \
.count()
if count > DIFFERENT_IP_LIMIT:
msg = ("Hourly access threshold exceeded for token %s\n\n"
"This token has been used from %s different IP "
"addresses during the last %s minutes.") % \
(access_token, count, CLEANUP_RANGE_MINUTES)
logging.info(msg)
# Checking if notification for admins about this token abuse has
# been sent in the last hour. This info is kept in cache.
key = "alert_sent_%s" % access_token
if not cache.get(key):
send_mail(
subject="[MetaBrainz] Hourly access threshold exceeded",
recipients=current_app.config['NOTIFICATION_RECIPIENTS'],
text=msg,
)
cache.set(key, True, 3600) # 1 hour
return new_record
def get_timestamps_for_user(self, user_name):
""" Return the max_ts and min_ts for a given user and cache the result in brainzutils cache
"""
tss = cache.get(REDIS_USER_TIMESTAMPS % user_name)
if tss:
(min_ts, max_ts) = tss.split(",")
min_ts = int(min_ts)
max_ts = int(max_ts)
else:
query = 'SELECT first(artist_msid) FROM ' + get_escaped_measurement_name(user_name)
min_ts = self._select_single_timestamp(query, get_measurement_name(user_name))
query = 'SELECT last(artist_msid) FROM ' + get_escaped_measurement_name(user_name)
max_ts = self._select_single_timestamp(query, get_measurement_name(user_name))
cache.set(REDIS_USER_TIMESTAMPS % user_name, "%d,%d" % (min_ts, max_ts), USER_CACHE_TIME)
return min_ts, max_ts
def lookup_ips(users):
""" Try to lookup and cache as many reverse DNS as possible in a window of time """
data = []
timeout = time.monotonic() + StatsView.IP_ADDR_TIMEOUT
for user in users:
row = list(user)
reverse = cache.get(user[0])
if not reverse:
if time.monotonic() < timeout:
reverse = StatsView.dns_lookup(user[0])
else:
reverse = None
if reverse:
cache.set(user[0], reverse, 3600)
row[0] = reverse
data.append(row)
return data
def get_listen_count_for_user(self, user_name, need_exact=False):
"""Get the total number of listens for a user. The number of listens comes from
brainzutils cache unless an exact number is asked for.
Args:
user_name: the user to get listens for
need_exact: if True, get an exact number of listens directly from the ListenStore
"""
if not need_exact:
# check if the user's listen count is already in cache
# if already present return it directly instead of calculating it again
# decode is set to False as we have not encoded the value when we set it
# in brainzutils cache as we need to call increment operation which requires
# an integer value
user_key = '{}{}'.format(REDIS_INFLUX_USER_LISTEN_COUNT, user_name)
count = cache.get(user_key, decode=False)
if count:
return int(count)
try:
results = self.influx.query('SELECT count(*) FROM ' + get_escaped_measurement_name(user_name))
except (InfluxDBServerError, InfluxDBClientError) as e:
self.log.error("Cannot query influx: %s" % str(e), exc_info=True)
raise
# get the number of listens from the json
try:
count = results.get_points(measurement = get_measurement_name(user_name)).__next__()['count_recording_msid']
except (KeyError, StopIteration):
count = 0
# put this value into brainzutils cache with an expiry time
user_key = "{}{}".format(REDIS_INFLUX_USER_LISTEN_COUNT, user_name)
cache.set(user_key, int(count), InfluxListenStore.USER_LISTEN_COUNT_CACHE_TIME, encode=False)
return int(count)
def test_increment_invalid_value(self):
cache.set("a", "not a number")
with self.assertRaises(redis.exceptions.ResponseError):
cache.increment("a")
def test_no_init(self):
cache._r = None
with self.assertRaises(RuntimeError):
cache.set("test", "testing")
with self.assertRaises(RuntimeError):
cache.get("test")
def test_single(self):
self.assertTrue(cache.set("test2", "Hello!"))
self.assertEqual(cache.get("test2"), "Hello!")
def test_single_no_encode(self):
self.assertTrue(cache.set("no encode", 1, encode=False))
self.assertEqual(cache.get("no encode", decode=False), b"1")
def test_single_with_namespace(self):
self.assertTrue(cache.set("test", 42, namespace="testing"))
self.assertEqual(cache.get("test", namespace="testing"), 42)
def test_single_fancy(self):
self.assertTrue(cache.set("test3", u"Привет!"))
self.assertEqual(cache.get("test3"), u"Привет!")
def get_popular(limit=None):
"""Get a list of popular reviews.
Popularity is determined by 'popularity' of a particular review. popularity is a
difference between positive votes and negative. In this case only votes
from the last month are used to calculate popularity to make results more
varied.
Args:
limit (int): Maximum number of reviews to return.
Returns:
Randomized list of popular reviews which are converted into
dictionaries using to_dict method.
"""
cache_key = cache.gen_key("popular_reviews", limit)
reviews = cache.get(cache_key, REVIEW_CACHE_NAMESPACE)
defined_limit = 4 * limit if limit else None
if not reviews:
with db.engine.connect() as connection:
results = connection.execute(sqlalchemy.text("""
SELECT review.id,
review.entity_id,
review.entity_type,
review.user_id,
review.edits,
review.is_draft,
review.is_hidden,
review.license_id,
review.language,
review.source,
review.source_url,
SUM(
CASE WHEN vote = 't' THEN 1
WHEN vote = 'f' THEN -1
WHEN vote IS NULL THEN 0 END
) AS popularity,
latest_revision.id AS latest_revision_id,
latest_revision.timestamp AS latest_revision_timestamp,
latest_revision.text AS text,
latest_revision.rating AS rating
FROM review
JOIN revision ON revision.review_id = review.id
LEFT JOIN (
SELECT revision_id, vote
FROM vote
WHERE rated_at > :last_month
) AS votes_last_month
ON votes_last_month.revision_id = revision.id
JOIN (
revision JOIN (
SELECT review.id AS review_uuid,
MAX(timestamp) AS latest_timestamp
FROM review
JOIN revision ON review.id = review_id
GROUP BY review.id
) AS latest ON latest.review_uuid = revision.review_id
AND latest.latest_timestamp = revision.timestamp
) AS latest_revision
ON review.id = latest_revision.review_id
WHERE entity_id
IN (
SELECT DISTINCT entity_id
FROM (
SELECT entity_id
FROM review
ORDER BY RANDOM()
) AS randomized_entity_ids
)
AND latest_revision.text IS NOT NULL
AND review.is_hidden = 'f'
AND review.is_draft = 'f'
GROUP BY review.id, latest_revision.id
ORDER BY popularity
LIMIT :limit
"""), {
"limit": defined_limit,
"last_month": datetime.now() - timedelta(weeks=4)
})
reviews = results.fetchall()
reviews = [dict(review) for review in reviews]
if reviews:
for review in reviews:
review["rating"] = RATING_SCALE_1_5.get(review["rating"])
review["last_revision"] = {
"id": review.pop("latest_revision_id"),
"timestamp": review.pop("latest_revision_timestamp"),
"text": review["text"],
"rating": review["rating"],
"review_id": review["id"],
}
reviews = [to_dict(review, confidential=True) for review in reviews]
cache.set(cache_key, reviews, 1 * 60 * 60, REVIEW_CACHE_NAMESPACE) # 1 hour
shuffle(reviews)
return reviews[:limit]
def test_increment(self):
cache.set("a", 1, encode=False)
self.assertEqual(cache.increment("a"), 2)
def test_delete(self):
key = "testing"
self.assertTrue(cache.set(key, u"Пример"))
self.assertEqual(cache.get(key), u"Пример")
self.assertEqual(cache.delete(key), 1)
self.assertIsNone(cache.get(key))
def review_list_handler():
"""Get list of reviews.
**Request Example:**
.. code-block:: bash
$ curl "https://critiquebrainz.org/ws/1/review/?limit=1&offset=50" \\
-X GET
**Response Example:**
.. code-block:: json
{
"count": 9197,
"limit": 1,
"offset": 50,
"reviews": [
{
"created": "Fri, 16 May 2008 00:00:00 GMT",
"edits": 0,
"entity_id": "09259937-6477-3959-8b10-af1cbaea8e6e",
"entity_type": "release_group",
"id": "c807d0b4-0dd0-43fe-a7c4-d29bb61f389e",
"language": "en",
"last_updated": "Fri, 16 May 2008 00:00:00 GMT",
"license": {
"full_name": "Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported",
"id": "CC BY-NC-SA 3.0",
"info_url": "https://creativecommons.org/licenses/by-nc-sa/3.0/"
},
"popularity": 0,
"source": "BBC",
"source_url": "http://www.bbc.co.uk/music/reviews/vh54",
"text": "TEXT CONTENT OF REVIEW",
"rating": 5,
"user": {
"created": "Wed, 07 May 2014 16:20:47 GMT",
"display_name": "Jenny Nelson",
"id": "3bf3fe0c-6db2-4746-bcf1-f39912113852",
"karma": 0,
"user_type": "Noob"
},
"votes": {
"positive": 0,
"negative": 0
}
}
]
}
:json uuid entity_id: UUID of the release group that is being reviewed
:json string entity_type: One of the supported reviewable entities. 'release_group' or 'event' etc. **(optional)**
:query user_id: user's UUID **(optional)**
:query sort: ``popularity`` or ``published_on`` **(optional)**
:query limit: results limit, min is 0, max is 50, default is 50 **(optional)**
:query offset: result offset, default is 0 **(optional)**
:query language: language code (ISO 639-1) **(optional)**
:resheader Content-Type: *application/json*
"""
# TODO: This checking is added to keep old clients working and needs to be removed.
release_group = Parser.uuid('uri', 'release_group', optional=True)
if release_group:
entity_id = release_group
entity_type = 'release_group'
else:
entity_id = Parser.uuid('uri', 'entity_id', optional=True)
entity_type = Parser.string('uri', 'entity_type', valid_values=ENTITY_TYPES, optional=True)
user_id = Parser.uuid('uri', 'user_id', optional=True)
# TODO: "rating" sort value is deprecated and needs to be removed.
sort = Parser.string('uri', 'sort', valid_values=['popularity', 'published_on', 'rating'], optional=True)
if sort == 'rating':
sort = 'popularity'
limit = Parser.int('uri', 'limit', min=1, max=50, optional=True) or 50
offset = Parser.int('uri', 'offset', optional=True) or 0
language = Parser.string('uri', 'language', min=2, max=3, optional=True)
if language and language not in supported_languages:
raise InvalidRequest(desc='Unsupported language')
# TODO(roman): Ideally caching logic should live inside the model. Otherwise it
# becomes hard to track all this stuff.
cache_key = cache.gen_key('list', entity_id, user_id, sort, limit, offset, language)
cached_result = cache.get(cache_key, REVIEW_CACHE_NAMESPACE)
if cached_result:
reviews = cached_result['reviews']
count = cached_result['count']
else:
reviews, count = db_review.list_reviews(
entity_id=entity_id,
entity_type=entity_type,
user_id=user_id,
sort=sort,
limit=limit,
offset=offset,
language=language,
)
reviews = [db_review.to_dict(p) for p in reviews]
cache.set(cache_key, {
'reviews': reviews,
'count': count,
}, namespace=REVIEW_CACHE_NAMESPACE)
return jsonify(limit=limit, offset=offset, count=count, reviews=reviews)