def test_user_page(self): response = self.client.get( url_for('user.profile', user_name=self.user['musicbrainz_id'])) self.assert200(response) self.assertContext('section', 'listens') # check that artist count is not shown if stats haven't been calculated yet response = self.client.get( url_for('user.profile', user_name=self.user['musicbrainz_id'])) self.assert200(response) self.assertTemplateUsed('user/profile.html') self.assertContext('artist_count', None) # check that artist count is shown if stats have been calculated db_stats.insert_user_stats( user_id=self.user['id'], artists={}, recordings={}, releases={}, artist_count=2, ) response = self.client.get( url_for('user.profile', user_name=self.user['musicbrainz_id'])) self.assert200(response) self.assertTemplateUsed('user/profile.html') self.assertContext('artist_count', '2')
def calculate_stats_for_user(self, user): """ Calculate statistics for specified user. Args: user: a dict which should contain the `id` and `musicbrainz_id` keys Returns: bool value specifying if user's stats were calculated or not. """ try: user = { 'id': user['id'], 'musicbrainz_id': user['musicbrainz_id'] } except KeyError: current_app.logger.error('Invalid user data sent into queue, ignoring...') return False # if this user already has recent stats, ignore if db_stats.valid_stats_exist(user['id']): current_app.logger.info('Stats already exist for user %s, moving on!', user['musicbrainz_id']) return False try: current_app.logger.info('Calculating statistics for user %s...', user['musicbrainz_id']) recordings = stats_user.get_top_recordings(self.bigquery, user['musicbrainz_id']) current_app.logger.info('Top recordings for user %s done!', user['musicbrainz_id']) artists = stats_user.get_top_artists(self.bigquery, user['musicbrainz_id']) current_app.logger.info('Top artists for user %s done!', user['musicbrainz_id']) releases = stats_user.get_top_releases(self.bigquery, user['musicbrainz_id']) current_app.logger.info('Top releases for user %s done!', user['musicbrainz_id']) artist_count = stats_user.get_artist_count(self.bigquery, user['musicbrainz_id']) current_app.logger.info('Artist count for user %s done!', user['musicbrainz_id']) except Exception as e: current_app.logger.error('Unable to calculate stats for user %s: %s', user['musicbrainz_id'], str(e), exc_info=True) raise current_app.logger.info('Inserting calculated stats for user %s into db', user['musicbrainz_id']) while True: try: db_stats.insert_user_stats( user_id=user['id'], artists=artists, recordings=recordings, releases=releases, artist_count=artist_count ) current_app.logger.info('Stats calculation for user %s done!', user['musicbrainz_id']) break except Exception as e: current_app.logger.error('Unable to insert calculated stats into db for user %s: %s', user['musicbrainz_id'], str(e), exc_info=True) time.sleep(3) return True
def test_top_artists(self): """ Tests the artist stats view """ # when no stats in db, it should redirect to the profile page r = self.client.get( url_for('user.artists', user_name=self.user['musicbrainz_id'])) self.assertRedirects( r, url_for('user.profile', user_name=self.user['musicbrainz_id'])) r = self.client.get(url_for('user.artists', user_name=self.user['musicbrainz_id']), follow_redirects=True) self.assert200(r) self.assertIn('No data calculated', r.data.decode('utf-8')) # add some artist stats to the db with open(self.path_to_data_file('user_top_artists.json')) as f: artists = ujson.load(f) db_stats.insert_user_stats( user_id=self.user['id'], artists=artists, recordings={}, releases={}, artist_count=2, ) r = self.client.get( url_for('user.artists', user_name=self.user['musicbrainz_id'])) self.assert200(r) self.assertContext('section', 'artists')
def test_insert_user_stats(self): with open(self.path_to_data_file('user_top_artists.json')) as f: artists = json.load(f) with open(self.path_to_data_file('user_top_releases.json')) as f: releases = json.load(f) with open(self.path_to_data_file('user_top_recordings.json')) as f: recordings = json.load(f) db_stats.insert_user_stats( user_id=self.user['id'], artists=artists, recordings=recordings, releases=releases, artist_count=2, ) result = db_stats.get_all_user_stats(user_id=self.user['id']) self.assertListEqual(result['artist']['all_time']['artists'], artists) self.assertEqual(result['artist']['count'], 2) self.assertListEqual(result['release']['all_time']['releases'], releases) self.assertListEqual(result['recording']['all_time']['recordings'], recordings) self.assertGreater(int(result['last_updated'].strftime('%s')), 0)
def test_top_artists(self): """ Tests the artist stats view """ # when no stats in db, it should redirect to the profile page r = self.client.get(url_for('user.artists', user_name=self.user.musicbrainz_id)) self.assertRedirects(r, url_for('user.profile', user_name=self.user.musicbrainz_id)) r = self.client.get(url_for('user.artists', user_name=self.user.musicbrainz_id), follow_redirects=True) self.assert200(r) self.assertIn('No data calculated', r.data.decode('utf-8')) # add some artist stats to the db with open(self.path_to_data_file('user_top_artists.json')) as f: artists = ujson.load(f) db_stats.insert_user_stats( user_id=self.user.id, artists=artists, recordings={}, releases={}, artist_count=2, ) r = self.client.get(url_for('user.artists', user_name=self.user.musicbrainz_id)) self.assert200(r) self.assertContext('section', 'artists')
def test_user_page(self): response = self.client.get( url_for('user.profile', user_name=self.user.musicbrainz_id)) self.assert200(response) self.assertContext('active_section', 'listens') # check that artist count is not shown if stats haven't been calculated yet response = self.client.get( url_for('user.profile', user_name=self.user.musicbrainz_id)) self.assert200(response) self.assertTemplateUsed('user/profile.html') props = ujson.loads(self.get_context_variable('props')) self.assertIsNone(props['artist_count']) # check that artist count is shown if stats have been calculated db_stats.insert_user_stats( user_id=self.user.id, artists={}, recordings={}, releases={}, artist_count=2, ) response = self.client.get( url_for('user.profile', user_name=self.user.musicbrainz_id)) self.assert200(response) self.assertTemplateUsed('user/profile.html') props = ujson.loads(self.get_context_variable('props')) self.assertEqual(props['artist_count'], '2') self.assertDictEqual(props['spotify'], {})
def handle_user_artist(data): """ Take artist stats for a user and save it in the database. """ musicbrainz_id = data['musicbrainz_id'] user = db_user.get_by_mb_id(musicbrainz_id) if not user: return artists = data['artist_stats'] artist_count = data['artist_count'] db_stats.insert_user_stats(user['id'], artists, {}, {}, artist_count)
def test_info_valid_stats(self): db_stats.insert_user_stats( user_id=self.user['id'], artists={}, recordings={}, releases={}, artist_count=0, ) self.temporary_login(self.user['id']) response = self.client.get(url_for('profile.info')) self.assert200(response) self.assertIn('Please wait until our next batch', str(response.data))
def handle_user_artist(data): """ Take artist stats for a user and save it in the database. """ musicbrainz_id = data['musicbrainz_id'] user = db_user.get_by_mb_id(musicbrainz_id) if not user: return # send a notification if this is a new batch of stats if is_new_user_stats_batch(): notify_user_stats_update() artists = data['artist_stats'] artist_count = data['artist_count'] db_stats.insert_user_stats(user['id'], artists, {}, {}, artist_count)
def test_top_artists(self): """ Tests the artist stats view """ # when no stats in db, it should redirect to the profile page r = self.client.get( url_for('user.artists', user_name=self.user.musicbrainz_id)) self.assertRedirects( r, url_for('user.profile', user_name=self.user.musicbrainz_id)) r = self.client.get(url_for('user.artists', user_name=self.user.musicbrainz_id), follow_redirects=True) self.assert200(r) self.assertIn('No data calculated', r.data.decode('utf-8')) # add some artist stats to the db with open(self.path_to_data_file('user_top_artists.json')) as f: artists = ujson.load(f) # insert empty documents to check for KeyError / ISE db_stats.insert_user_stats( user_id=self.user.id, artists={}, recordings={}, releases={}, artist_count=2, yearmonth='2019-01', ) r = self.client.get( url_for('user.artists', user_name=self.user.musicbrainz_id)) self.assert200(r) self.assertContext('active_section', 'artists') db_stats.insert_user_stats( user_id=self.user.id, artists=artists, recordings={}, releases={}, artist_count=2, yearmonth='2019-01', ) r = self.client.get( url_for('user.artists', user_name=self.user.musicbrainz_id)) self.assert200(r) self.assertContext('active_section', 'artists')
def handle_user_artist(data): """ Take artist stats for a user and save it in the database. """ musicbrainz_id = data['musicbrainz_id'] user = db_user.get_by_mb_id(musicbrainz_id) if not user: current_app.logger.critical( "Calculated stats for a user that doesn't exist in the Postgres database: %s", musicbrainz_id) return # send a notification if this is a new batch of stats if is_new_user_stats_batch(): notify_user_stats_update() current_app.logger.debug("inserting stats for user %s", musicbrainz_id) artists = data['artist_stats'] artist_count = data['artist_count'] db_stats.insert_user_stats(user['id'], artists, {}, {}, artist_count)
def test_delete(self): user_id = db_user.create(10, 'frank') user = db_user.get(user_id) self.assertIsNotNone(user) db_stats.insert_user_stats( user_id=user_id, artists={}, recordings={}, releases={}, artist_count=2, ) user_stats = db_stats.get_all_user_stats(user_id) self.assertIsNotNone(user_stats) db_user.delete(user_id) user = db_user.get(user_id) self.assertIsNone(user) user_stats = db_stats.get_all_user_stats(user_id) self.assertIsNone(user_stats)
def test_request_stats(self, mock_publish): self.temporary_login(self.user['id']) response = self.client.get(url_for('profile.request_stats'), follow_redirects=True) self.assertStatus(response, 200) self.assertIn('You have been added to the stats calculation queue', str(response.data)) db_stats.insert_user_stats( user_id=self.user['id'], artists={}, recordings={}, releases={}, artist_count=0, ) response = self.client.get(url_for('profile.request_stats'), follow_redirects=True) self.assertStatus(response, 200) self.assertIn('please wait until the next interval', str(response.data))
def callback(self, ch, method, properties, body): """ Handle the data received from the queue and insert into the database accordingly. """ data = ujson.loads(body) for username, metadata in data.items(): user = db_user.get_by_mb_id(username) if not user: break artists = metadata['artists']['artist_stats'] artist_count = metadata['artists']['artist_count'] db_stats.insert_user_stats(user['id'], artists, {}, {}, artist_count) current_app.logger.info("data for {} published".format(username)) while True: try: self.incoming_ch.basic_ack(delivery_tag=method.delivery_tag) break except pika.exceptions.ConnectionClosed: self.init_rabbitmq_connection()
def test_user_page(self): response = self.client.get(url_for('user.profile', user_name=self.user.musicbrainz_id)) self.assert200(response) self.assertContext('section', 'listens') # check that artist count is not shown if stats haven't been calculated yet response = self.client.get(url_for('user.profile', user_name=self.user.musicbrainz_id)) self.assert200(response) self.assertTemplateUsed('user/profile.html') self.assertContext('artist_count', None) # check that artist count is shown if stats have been calculated db_stats.insert_user_stats( user_id=self.user.id, artists={}, recordings={}, releases={}, artist_count=2, ) response = self.client.get(url_for('user.profile', user_name=self.user.musicbrainz_id)) self.assert200(response) self.assertTemplateUsed('user/profile.html') self.assertContext('artist_count', '2')
def insert_test_data(self): """ Insert test data into the database """ with open(self.path_to_data_file('user_top_artists.json')) as f: artists = json.load(f) with open(self.path_to_data_file('user_top_releases.json')) as f: releases = json.load(f) with open(self.path_to_data_file('user_top_recordings.json')) as f: recordings = json.load(f) db_stats.insert_user_stats( user_id=self.user['id'], artists=artists, recordings=recordings, releases=releases, artist_count=2, ) return { 'user_artists': artists, 'user_releases': releases, 'user_recordings': recordings, }
def test_insert_user_stats(self): with open(self.path_to_data_file('user_top_artists.json')) as f: artists = json.load(f) with open(self.path_to_data_file('user_top_releases.json')) as f: releases = json.load(f) with open(self.path_to_data_file('user_top_recordings.json')) as f: recordings = json.load(f) db_stats.insert_user_stats( user_id=self.user['id'], artists=artists, recordings=recordings, releases=releases, artist_count=2, ) result = db_stats.get_all_user_stats(user_id=self.user['id']) self.assertListEqual(result['artist']['all_time'], artists) self.assertEqual(result['artist']['count'], 2) self.assertListEqual(result['release']['all_time'], releases) self.assertListEqual(result['recording']['all_time'], recordings) self.assertGreater(int(result['last_updated'].strftime('%s')), 0)