def test_insert_user_stats_mult_ranges_artist_map(self): """ Test if multiple time range data is inserted correctly """ with open(self.path_to_data_file('user_artist_map_db.json')) as f: artist_map_data = json.load(f) artist_map_data_year = deepcopy(artist_map_data) artist_map_data_year['stats_range'] = 'year' db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='artist_map', stats=StatRange[UserArtistMapRecord](**artist_map_data)) db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='artist_map', stats=StatRange[UserArtistMapRecord](**artist_map_data_year)) result = db_stats.get_user_artist_map(1, 'all_time') self.assertDictEqual( result.dict(exclude={'user_id', 'last_updated', 'count'}), artist_map_data) result = db_stats.get_user_artist_map(1, 'year') self.assertDictEqual( result.dict(exclude={'user_id', 'last_updated', 'count'}), artist_map_data_year)
def test_insert_user_stats_mult_ranges_daily_activity(self): """ Test if multiple time range data is inserted correctly """ with open(self.path_to_data_file('user_daily_activity_db.json')) as f: daily_activity_data = json.load(f) daily_activity_data_year = deepcopy(daily_activity_data) daily_activity_data_year['stats_range'] = 'year' db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='daily_activity', stats=StatRange[UserDailyActivityRecord](**daily_activity_data)) db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='daily_activity', stats=StatRange[UserDailyActivityRecord]( **daily_activity_data_year)) result = db_stats.get_user_daily_activity(1, 'all_time') self.assertDictEqual( result.dict(exclude={'user_id', 'last_updated', 'count'}), daily_activity_data) result = db_stats.get_user_daily_activity(1, 'year') self.assertDictEqual( result.dict(exclude={'user_id', 'last_updated', 'count'}), daily_activity_data_year)
def test_insert_user_stats_mult_ranges_recording(self): """ Test if multiple time range data is inserted correctly """ with open(self.path_to_data_file('user_top_recordings_db.json')) as f: recordings_data = json.load(f) recordings_data_year = deepcopy(recordings_data) recordings_data_year['stats_range'] = 'year' db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='recordings', stats=StatRange[UserEntityRecord](**recordings_data)) db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='recordings', stats=StatRange[UserEntityRecord](**recordings_data_year)) result = db_stats.get_user_stats(user_id=self.user['id'], stats_range='all_time', stats_type='recordings') self.assertDictEqual(result.dict(exclude={'user_id', 'last_updated'}), recordings_data) result = db_stats.get_user_stats(user_id=self.user['id'], stats_range='year', stats_type='recordings') self.assertDictEqual(result.dict(exclude={'user_id', 'last_updated'}), recordings_data_year)
def test_insert_user_listening_activity(self): """ Test if listening activity stats are inserted correctly """ with open(self.path_to_data_file('user_listening_activity_db.json')) as f: listening_activity_data = json.load(f) db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='listening_activity', stats=StatRange[ListeningActivityRecord](**listening_activity_data) )
def test_insert_user_recordings(self): """ Test if recording stats are inserted correctly """ with open(self.path_to_data_file('user_top_recordings_db.json')) as f: recordings_data = json.load(f) db_stats.insert_user_jsonb_data(user_id=self.user['id'], stats_type='recordings', stats=StatRange[EntityRecord](**recordings_data)) result = db_stats.get_user_stats(user_id=self.user['id'], stats_range='all_time', stats_type='recordings') self.assertDictEqual(result.dict(exclude={'user_id', 'last_updated'}), recordings_data)
def test_insert_user_artist_map(self): """ Test if daily activity stats are inserted correctly """ with open(self.path_to_data_file('user_artist_map_db.json')) as f: artist_map_data = json.load(f) db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='artist_map', stats=StatRange[UserArtistMapRecord](**artist_map_data) ) result = db_stats.get_user_artist_map(user_id=self.user['id'], stats_range='all_time') self.assertDictEqual(result.dict(exclude={'user_id', 'last_updated', 'count'}), artist_map_data)
def test_delete(self): user_id = db_user.create(10, 'frank') user = db_user.get(user_id) self.assertIsNotNone(user) with open(self.path_to_data_file('user_top_artists_db.json')) as f: artists_data = ujson.load(f) db_stats.insert_user_jsonb_data( user_id=user_id, stats_type='artists', stats=StatRange[EntityRecord](**artists_data), ) user_stats = db_stats.get_user_stats(user_id, 'all_time', 'artists') self.assertIsNotNone(user_stats) db_user.delete(user_id) user = db_user.get(user_id) self.assertIsNone(user) user_stats = db_stats.get_user_stats(user_id, 'all_time', 'artists') self.assertIsNone(user_stats)
def _handle_user_activity_stats(stats_type, stats_model, data): musicbrainz_id = data['musicbrainz_id'] user = db_user.get_by_mb_id(musicbrainz_id) if not user: current_app.logger.info( "Calculated stats for a user that doesn't exist in the Postgres database: %s", musicbrainz_id) return # send a notification if this is a new batch of stats if is_new_user_stats_batch(): notify_user_stats_update(stat_type=data.get('type', '')) current_app.logger.debug("inserting stats for user %s", musicbrainz_id) stats_range = data['stats_range'] try: db_stats.insert_user_jsonb_data(user['id'], stats_type, stats_model(**data)) except ValidationError: current_app.logger.error( f"""ValidationError while inserting {stats_range} {stats_type} for user with user_id: {user['id']}. Data: {json.dumps(data, indent=3)}""", exc_info=True)
def handle_user_entity(data): """ Take entity stats for a user and save it in the database. """ musicbrainz_id = data['musicbrainz_id'] user = db_user.get_by_mb_id(musicbrainz_id) if not user: return # send a notification if this is a new batch of stats if is_new_user_stats_batch(): notify_user_stats_update(stat_type=data.get('type', '')) current_app.logger.debug("inserting stats for user %s", musicbrainz_id) stats_range = data['stats_range'] entity = data['entity'] try: db_stats.insert_user_jsonb_data(user['id'], entity, StatRange[UserEntityRecord](**data)) except ValidationError: current_app.logger.error( f"""ValidationError while inserting {stats_range} top {entity} for user with user_id: {user['id']}. Data: {json.dumps({stats_range: data}, indent=3)}""", exc_info=True)
def test_user_page(self): response = self.client.get(url_for('user.profile', user_name=self.user.musicbrainz_id)) self.assert200(response) self.assertContext('active_section', 'listens') # check that artist count is not shown if stats haven't been calculated yet response = self.client.get(url_for('user.profile', user_name=self.user.musicbrainz_id)) self.assert200(response) self.assertTemplateUsed('user/profile.html') props = ujson.loads(self.get_context_variable('props')) self.assertIsNone(props['artist_count']) with open(self.path_to_data_file('user_top_artists_db.json')) as f: artists_data = ujson.load(f) db_stats.insert_user_jsonb_data(user_id=self.user.id, stats_type='artists', stats=StatRange[UserEntityRecord](**artists_data)) response = self.client.get(url_for('user.profile', user_name=self.user.musicbrainz_id)) self.assert200(response) self.assertTemplateUsed('user/profile.html') props = ujson.loads(self.get_context_variable('props')) self.assertEqual(props['artist_count'], '2') global_props = ujson.loads(self.get_context_variable("global_props")) self.assertDictEqual(global_props['spotify'], {})
def _get_artist_map_stats(user_id, stats_range): recalculate_param = request.args.get('force_recalculate', default='false') if recalculate_param.lower() not in ['true', 'false']: raise APIBadRequest( "Invalid value of force_recalculate: {}".format(recalculate_param)) force_recalculate = recalculate_param.lower() == 'true' # Check if stats are present in DB, if not calculate them calculated = not force_recalculate stats = db_stats.get_user_artist_map(user_id, stats_range) if stats is None: calculated = False # Check if the stats present in DB have been calculated in the past week, if not recalculate them stale = False if calculated: if (datetime.now(timezone.utc) - stats.last_updated).days >= STATS_CALCULATION_INTERVAL: stale = True if stale or not calculated: artist_stats = db_stats.get_user_stats(user_id, stats_range, 'artists') # If top artists are missing, return the stale stats if present, otherwise return 204 if artist_stats is None: if stale: result = stats else: raise APINoContent('') else: # Calculate the data artist_mbid_counts = defaultdict(int) for artist in artist_stats.data.__root__: for artist_mbid in artist.artist_mbids: artist_mbid_counts[artist_mbid] += artist.listen_count country_code_data = _get_country_wise_counts(artist_mbid_counts) result = StatApi[UserArtistMapRecord]( **{ "data": country_code_data, "from_ts": artist_stats.from_ts, "to_ts": artist_stats.to_ts, "stats_range": stats_range, # this isn't stored in the database, but adding it here to avoid a subsequent db call to # just fetch last updated. could just store this value instead in db but that complicates # the implementation a bit "last_updated": datetime.now(), "user_id": user_id }) # Store in DB for future use try: db_stats.insert_user_jsonb_data(user_id, 'artist_map', result) except Exception as err: current_app.logger.error( "Error while inserting artist map stats for {user}. Error: {err}. Data: {data}" .format(user=user_id, err=err, data=result), exc_info=True) else: result = stats return result
def get_artist_map(user_name: str): """ Get the artist map for user ``user_name``. The artist map shows the number of artists the user has listened to from different countries of the world. A sample response from the endpoint may look like: .. code-block:: json { "payload": { "from_ts": 1587945600, "last_updated": 1592807084, "artist_map": [ { "country": "USA", "artist_count": 34 }, { "country": "GBR", "artist_count": 69 }, { "country": "IND", "artist_count": 32 } ], "stats_range": "all_time" "to_ts": 1589155200, "user_id": "ishaanshah" } } .. note:: - This endpoint is currently in beta - We cache the results for this query for a week to improve page load times, if you want to request fresh data you can use the ``force_recalculate`` flag. :param range: Optional, time interval for which statistics should be returned, possible values are ``week``, ``month``, ``year``, ``all_time``, defaults to ``all_time`` :type range: ``str`` :param force_recalculate: Optional, recalculate the data instead of returning the cached result. :type force_recalculate: ``bool`` :statuscode 200: Successful query, you have data! :statuscode 204: Statistics for the user haven't been calculated, empty response will be returned :statuscode 400: Bad request, check ``response['error']`` for more details :statuscode 404: User not found :resheader Content-Type: *application/json* """ user, stats_range = _validate_stats_user_params(user_name) recalculate_param = request.args.get('force_recalculate', default='false') if recalculate_param.lower() not in ['true', 'false']: raise APIBadRequest( "Invalid value of force_recalculate: {}".format(recalculate_param)) force_recalculate = recalculate_param.lower() == 'true' # Check if stats are present in DB, if not calculate them calculated = not force_recalculate stats = db_stats.get_user_artist_map(user['id'], stats_range) if stats is None: calculated = False # Check if the stats present in DB have been calculated in the past week, if not recalculate them stale = False if calculated: if (datetime.now(timezone.utc) - stats.last_updated).days >= STATS_CALCULATION_INTERVAL: stale = True if stale or not calculated: artist_stats = db_stats.get_user_stats(user['id'], stats_range, 'artists') # If top artists are missing, return the stale stats if present, otherwise return 204 if artist_stats is None: if stale: result = stats else: raise APINoContent('') else: # Calculate the data artist_mbid_counts = defaultdict(int) for artist in artist_stats.data.__root__: for artist_mbid in artist.artist_mbids: artist_mbid_counts[artist_mbid] += artist.listen_count country_code_data = _get_country_wise_counts(artist_mbid_counts) result = StatApi[UserArtistMapRecord]( **{ "data": country_code_data, "from_ts": artist_stats.from_ts, "to_ts": artist_stats.to_ts, "stats_range": stats_range, # this isn't stored in the database, but adding it here to avoid a subsequent db call to # just fetch last updated. could just store this value instead in db but that complicates # the implementation a bit "last_updated": datetime.now(), "user_id": user['id'] }) # Store in DB for future use try: db_stats.insert_user_jsonb_data(user['id'], 'artist_map', result) except Exception as err: current_app.logger.error( "Error while inserting artist map stats for {user}. Error: {err}. Data: {data}" .format(user=user_name, err=err, data=result), exc_info=True) else: result = stats return jsonify({ "payload": { "user_id": user_name, "range": stats_range, "from_ts": result.from_ts, "to_ts": result.to_ts, "last_updated": int(result.last_updated.timestamp()), "artist_map": [x.dict() for x in result.data.__root__] } })
def insert_test_data(self): """ Insert test data into the database """ with open(self.path_to_data_file('user_top_artists_db.json')) as f: user_artists = json.load(f) with open(self.path_to_data_file('user_top_releases_db.json')) as f: releases = json.load(f) with open(self.path_to_data_file('user_top_recordings_db.json')) as f: recordings = json.load(f) with open(self.path_to_data_file('user_listening_activity_db.json')) as f: listening_activity = json.load(f) with open(self.path_to_data_file('user_daily_activity_db.json')) as f: daily_activity = json.load(f) with open(self.path_to_data_file('user_artist_map_db.json')) as f: artist_map = json.load(f) with open(self.path_to_data_file('sitewide_top_artists_db.json')) as f: sitewide_artists = json.load(f) db_stats.insert_user_jsonb_data(user_id=self.user['id'], stats_type='artists', stats=StatRange[EntityRecord](**user_artists)) db_stats.insert_user_jsonb_data(user_id=self.user['id'], stats_type='releases', stats=StatRange[EntityRecord](**releases)) db_stats.insert_user_jsonb_data(user_id=self.user['id'], stats_type='recordings', stats=StatRange[EntityRecord](**recordings)) db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='listening_activity', stats=StatRange[ListeningActivityRecord](**listening_activity) ) db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='daily_activity', stats=StatRange[DailyActivityRecord](**daily_activity) ) db_stats.insert_user_jsonb_data( user_id=self.user['id'], stats_type='artist_map', stats=StatRange[UserArtistMapRecord](**artist_map) ) db_stats.insert_sitewide_jsonb_data('artists', StatRange[EntityRecord](**sitewide_artists)) return { 'user_artists': user_artists, 'user_releases': releases, 'user_recordings': recordings, 'user_listening_activity': listening_activity, 'user_daily_activity': daily_activity, 'user_artist_map': artist_map, 'sitewide_artists': sitewide_artists }