def test_insert_user_stats_mult_ranges_artist_map(self):
""" Test if multiple time range data is inserted correctly """
with open(self.path_to_data_file('user_artist_map_db.json')) as f:
artist_map_data = json.load(f)
artist_map_data_year = deepcopy(artist_map_data)
artist_map_data_year['stats_range'] = 'year'
db_stats.insert_user_jsonb_data(
user_id=self.user['id'],
stats_type='artist_map',
stats=StatRange[UserArtistMapRecord](**artist_map_data))
db_stats.insert_user_jsonb_data(
user_id=self.user['id'],
stats_type='artist_map',
stats=StatRange[UserArtistMapRecord](**artist_map_data_year))
result = db_stats.get_user_artist_map(1, 'all_time')
self.assertDictEqual(
result.dict(exclude={'user_id', 'last_updated', 'count'}),
artist_map_data)
result = db_stats.get_user_artist_map(1, 'year')
self.assertDictEqual(
result.dict(exclude={'user_id', 'last_updated', 'count'}),
artist_map_data_year)
def test_insert_user_stats_mult_ranges_daily_activity(self):
""" Test if multiple time range data is inserted correctly """
with open(self.path_to_data_file('user_daily_activity_db.json')) as f:
daily_activity_data = json.load(f)
daily_activity_data_year = deepcopy(daily_activity_data)
daily_activity_data_year['stats_range'] = 'year'
db_stats.insert_user_jsonb_data(
user_id=self.user['id'],
stats_type='daily_activity',
stats=StatRange[UserDailyActivityRecord](**daily_activity_data))
db_stats.insert_user_jsonb_data(
user_id=self.user['id'],
stats_type='daily_activity',
stats=StatRange[UserDailyActivityRecord](
**daily_activity_data_year))
result = db_stats.get_user_daily_activity(1, 'all_time')
self.assertDictEqual(
result.dict(exclude={'user_id', 'last_updated', 'count'}),
daily_activity_data)
result = db_stats.get_user_daily_activity(1, 'year')
self.assertDictEqual(
result.dict(exclude={'user_id', 'last_updated', 'count'}),
daily_activity_data_year)
def test_insert_user_stats_mult_ranges_recording(self):
""" Test if multiple time range data is inserted correctly """
with open(self.path_to_data_file('user_top_recordings_db.json')) as f:
recordings_data = json.load(f)
recordings_data_year = deepcopy(recordings_data)
recordings_data_year['stats_range'] = 'year'
db_stats.insert_user_jsonb_data(
user_id=self.user['id'],
stats_type='recordings',
stats=StatRange[UserEntityRecord](**recordings_data))
db_stats.insert_user_jsonb_data(
user_id=self.user['id'],
stats_type='recordings',
stats=StatRange[UserEntityRecord](**recordings_data_year))
result = db_stats.get_user_stats(user_id=self.user['id'],
stats_range='all_time',
stats_type='recordings')
self.assertDictEqual(result.dict(exclude={'user_id', 'last_updated'}),
recordings_data)
result = db_stats.get_user_stats(user_id=self.user['id'],
stats_range='year',
stats_type='recordings')
self.assertDictEqual(result.dict(exclude={'user_id', 'last_updated'}),
recordings_data_year)
def test_insert_user_listening_activity(self):
""" Test if listening activity stats are inserted correctly """
with open(self.path_to_data_file('user_listening_activity_db.json')) as f:
listening_activity_data = json.load(f)
db_stats.insert_user_jsonb_data(
user_id=self.user['id'], stats_type='listening_activity',
stats=StatRange[ListeningActivityRecord](**listening_activity_data)
)
def test_insert_user_recordings(self):
""" Test if recording stats are inserted correctly """
with open(self.path_to_data_file('user_top_recordings_db.json')) as f:
recordings_data = json.load(f)
db_stats.insert_user_jsonb_data(user_id=self.user['id'], stats_type='recordings',
stats=StatRange[EntityRecord](**recordings_data))
result = db_stats.get_user_stats(user_id=self.user['id'], stats_range='all_time', stats_type='recordings')
self.assertDictEqual(result.dict(exclude={'user_id', 'last_updated'}), recordings_data)
def test_insert_user_artist_map(self):
""" Test if daily activity stats are inserted correctly """
with open(self.path_to_data_file('user_artist_map_db.json')) as f:
artist_map_data = json.load(f)
db_stats.insert_user_jsonb_data(
user_id=self.user['id'], stats_type='artist_map',
stats=StatRange[UserArtistMapRecord](**artist_map_data)
)
result = db_stats.get_user_artist_map(user_id=self.user['id'], stats_range='all_time')
self.assertDictEqual(result.dict(exclude={'user_id', 'last_updated', 'count'}), artist_map_data)
def test_delete(self):
user_id = db_user.create(10, 'frank')
user = db_user.get(user_id)
self.assertIsNotNone(user)
with open(self.path_to_data_file('user_top_artists_db.json')) as f:
artists_data = ujson.load(f)
db_stats.insert_user_jsonb_data(
user_id=user_id,
stats_type='artists',
stats=StatRange[EntityRecord](**artists_data),
)
user_stats = db_stats.get_user_stats(user_id, 'all_time', 'artists')
self.assertIsNotNone(user_stats)
db_user.delete(user_id)
user = db_user.get(user_id)
self.assertIsNone(user)
user_stats = db_stats.get_user_stats(user_id, 'all_time', 'artists')
self.assertIsNone(user_stats)
def _handle_user_activity_stats(stats_type, stats_model, data):
musicbrainz_id = data['musicbrainz_id']
user = db_user.get_by_mb_id(musicbrainz_id)
if not user:
current_app.logger.info(
"Calculated stats for a user that doesn't exist in the Postgres database: %s",
musicbrainz_id)
return
# send a notification if this is a new batch of stats
if is_new_user_stats_batch():
notify_user_stats_update(stat_type=data.get('type', ''))
current_app.logger.debug("inserting stats for user %s", musicbrainz_id)
stats_range = data['stats_range']
try:
db_stats.insert_user_jsonb_data(user['id'], stats_type,
stats_model(**data))
except ValidationError:
current_app.logger.error(
f"""ValidationError while inserting {stats_range} {stats_type} for
user with user_id: {user['id']}. Data: {json.dumps(data, indent=3)}""",
exc_info=True)
def handle_user_entity(data):
""" Take entity stats for a user and save it in the database. """
musicbrainz_id = data['musicbrainz_id']
user = db_user.get_by_mb_id(musicbrainz_id)
if not user:
return
# send a notification if this is a new batch of stats
if is_new_user_stats_batch():
notify_user_stats_update(stat_type=data.get('type', ''))
current_app.logger.debug("inserting stats for user %s", musicbrainz_id)
stats_range = data['stats_range']
entity = data['entity']
try:
db_stats.insert_user_jsonb_data(user['id'], entity,
StatRange[UserEntityRecord](**data))
except ValidationError:
current_app.logger.error(
f"""ValidationError while inserting {stats_range} top {entity} for user
with user_id: {user['id']}. Data: {json.dumps({stats_range: data}, indent=3)}""",
exc_info=True)
def test_user_page(self):
response = self.client.get(url_for('user.profile', user_name=self.user.musicbrainz_id))
self.assert200(response)
self.assertContext('active_section', 'listens')
# check that artist count is not shown if stats haven't been calculated yet
response = self.client.get(url_for('user.profile', user_name=self.user.musicbrainz_id))
self.assert200(response)
self.assertTemplateUsed('user/profile.html')
props = ujson.loads(self.get_context_variable('props'))
self.assertIsNone(props['artist_count'])
with open(self.path_to_data_file('user_top_artists_db.json')) as f:
artists_data = ujson.load(f)
db_stats.insert_user_jsonb_data(user_id=self.user.id, stats_type='artists',
stats=StatRange[UserEntityRecord](**artists_data))
response = self.client.get(url_for('user.profile', user_name=self.user.musicbrainz_id))
self.assert200(response)
self.assertTemplateUsed('user/profile.html')
props = ujson.loads(self.get_context_variable('props'))
self.assertEqual(props['artist_count'], '2')
global_props = ujson.loads(self.get_context_variable("global_props"))
self.assertDictEqual(global_props['spotify'], {})
def _get_artist_map_stats(user_id, stats_range):
recalculate_param = request.args.get('force_recalculate', default='false')
if recalculate_param.lower() not in ['true', 'false']:
raise APIBadRequest(
"Invalid value of force_recalculate: {}".format(recalculate_param))
force_recalculate = recalculate_param.lower() == 'true'
# Check if stats are present in DB, if not calculate them
calculated = not force_recalculate
stats = db_stats.get_user_artist_map(user_id, stats_range)
if stats is None:
calculated = False
# Check if the stats present in DB have been calculated in the past week, if not recalculate them
stale = False
if calculated:
if (datetime.now(timezone.utc) -
stats.last_updated).days >= STATS_CALCULATION_INTERVAL:
stale = True
if stale or not calculated:
artist_stats = db_stats.get_user_stats(user_id, stats_range, 'artists')
# If top artists are missing, return the stale stats if present, otherwise return 204
if artist_stats is None:
if stale:
result = stats
else:
raise APINoContent('')
else:
# Calculate the data
artist_mbid_counts = defaultdict(int)
for artist in artist_stats.data.__root__:
for artist_mbid in artist.artist_mbids:
artist_mbid_counts[artist_mbid] += artist.listen_count
country_code_data = _get_country_wise_counts(artist_mbid_counts)
result = StatApi[UserArtistMapRecord](
**{
"data": country_code_data,
"from_ts": artist_stats.from_ts,
"to_ts": artist_stats.to_ts,
"stats_range": stats_range,
# this isn't stored in the database, but adding it here to avoid a subsequent db call to
# just fetch last updated. could just store this value instead in db but that complicates
# the implementation a bit
"last_updated": datetime.now(),
"user_id": user_id
})
# Store in DB for future use
try:
db_stats.insert_user_jsonb_data(user_id, 'artist_map', result)
except Exception as err:
current_app.logger.error(
"Error while inserting artist map stats for {user}. Error: {err}. Data: {data}"
.format(user=user_id, err=err, data=result),
exc_info=True)
else:
result = stats
return result
def get_artist_map(user_name: str):
"""
Get the artist map for user ``user_name``. The artist map shows the number of artists the user has listened to
from different countries of the world.
A sample response from the endpoint may look like:
.. code-block:: json
{
"payload": {
"from_ts": 1587945600,
"last_updated": 1592807084,
"artist_map": [
{
"country": "USA",
"artist_count": 34
},
{
"country": "GBR",
"artist_count": 69
},
{
"country": "IND",
"artist_count": 32
}
],
"stats_range": "all_time"
"to_ts": 1589155200,
"user_id": "ishaanshah"
}
}
.. note::
- This endpoint is currently in beta
- We cache the results for this query for a week to improve page load times, if you want to request fresh data you
can use the ``force_recalculate`` flag.
:param range: Optional, time interval for which statistics should be returned, possible values are ``week``,
``month``, ``year``, ``all_time``, defaults to ``all_time``
:type range: ``str``
:param force_recalculate: Optional, recalculate the data instead of returning the cached result.
:type force_recalculate: ``bool``
:statuscode 200: Successful query, you have data!
:statuscode 204: Statistics for the user haven't been calculated, empty response will be returned
:statuscode 400: Bad request, check ``response['error']`` for more details
:statuscode 404: User not found
:resheader Content-Type: *application/json*
"""
user, stats_range = _validate_stats_user_params(user_name)
recalculate_param = request.args.get('force_recalculate', default='false')
if recalculate_param.lower() not in ['true', 'false']:
raise APIBadRequest(
"Invalid value of force_recalculate: {}".format(recalculate_param))
force_recalculate = recalculate_param.lower() == 'true'
# Check if stats are present in DB, if not calculate them
calculated = not force_recalculate
stats = db_stats.get_user_artist_map(user['id'], stats_range)
if stats is None:
calculated = False
# Check if the stats present in DB have been calculated in the past week, if not recalculate them
stale = False
if calculated:
if (datetime.now(timezone.utc) -
stats.last_updated).days >= STATS_CALCULATION_INTERVAL:
stale = True
if stale or not calculated:
artist_stats = db_stats.get_user_stats(user['id'], stats_range,
'artists')
# If top artists are missing, return the stale stats if present, otherwise return 204
if artist_stats is None:
if stale:
result = stats
else:
raise APINoContent('')
else:
# Calculate the data
artist_mbid_counts = defaultdict(int)
for artist in artist_stats.data.__root__:
for artist_mbid in artist.artist_mbids:
artist_mbid_counts[artist_mbid] += artist.listen_count
country_code_data = _get_country_wise_counts(artist_mbid_counts)
result = StatApi[UserArtistMapRecord](
**{
"data": country_code_data,
"from_ts": artist_stats.from_ts,
"to_ts": artist_stats.to_ts,
"stats_range": stats_range,
# this isn't stored in the database, but adding it here to avoid a subsequent db call to
# just fetch last updated. could just store this value instead in db but that complicates
# the implementation a bit
"last_updated": datetime.now(),
"user_id": user['id']
})
# Store in DB for future use
try:
db_stats.insert_user_jsonb_data(user['id'], 'artist_map',
result)
except Exception as err:
current_app.logger.error(
"Error while inserting artist map stats for {user}. Error: {err}. Data: {data}"
.format(user=user_name, err=err, data=result),
exc_info=True)
else:
result = stats
return jsonify({
"payload": {
"user_id": user_name,
"range": stats_range,
"from_ts": result.from_ts,
"to_ts": result.to_ts,
"last_updated": int(result.last_updated.timestamp()),
"artist_map": [x.dict() for x in result.data.__root__]
}
})
def insert_test_data(self):
""" Insert test data into the database """
with open(self.path_to_data_file('user_top_artists_db.json')) as f:
user_artists = json.load(f)
with open(self.path_to_data_file('user_top_releases_db.json')) as f:
releases = json.load(f)
with open(self.path_to_data_file('user_top_recordings_db.json')) as f:
recordings = json.load(f)
with open(self.path_to_data_file('user_listening_activity_db.json')) as f:
listening_activity = json.load(f)
with open(self.path_to_data_file('user_daily_activity_db.json')) as f:
daily_activity = json.load(f)
with open(self.path_to_data_file('user_artist_map_db.json')) as f:
artist_map = json.load(f)
with open(self.path_to_data_file('sitewide_top_artists_db.json')) as f:
sitewide_artists = json.load(f)
db_stats.insert_user_jsonb_data(user_id=self.user['id'], stats_type='artists',
stats=StatRange[EntityRecord](**user_artists))
db_stats.insert_user_jsonb_data(user_id=self.user['id'], stats_type='releases',
stats=StatRange[EntityRecord](**releases))
db_stats.insert_user_jsonb_data(user_id=self.user['id'], stats_type='recordings',
stats=StatRange[EntityRecord](**recordings))
db_stats.insert_user_jsonb_data(
user_id=self.user['id'], stats_type='listening_activity',
stats=StatRange[ListeningActivityRecord](**listening_activity)
)
db_stats.insert_user_jsonb_data(
user_id=self.user['id'], stats_type='daily_activity',
stats=StatRange[DailyActivityRecord](**daily_activity)
)
db_stats.insert_user_jsonb_data(
user_id=self.user['id'], stats_type='artist_map',
stats=StatRange[UserArtistMapRecord](**artist_map)
)
db_stats.insert_sitewide_jsonb_data('artists', StatRange[EntityRecord](**sitewide_artists))
return {
'user_artists': user_artists,
'user_releases': releases,
'user_recordings': recordings,
'user_listening_activity': listening_activity,
'user_daily_activity': daily_activity,
'user_artist_map': artist_map,
'sitewide_artists': sitewide_artists
}
