def test_index_aggregate_user_same_checkpoint(app): """Test that we should not update when last index is the same""" with app.app_context(): db = get_db() entities = deepcopy(basic_entities) current_blocknumber = basic_entities["blocks"][0]["number"] entities.update({ "indexing_checkpoints": [{ "tablename": AGGREGATE_USER, "last_checkpoint": current_blocknumber }], }) populate_mock_db(db, entities) with db.scoped_session() as session: results: List[AggregateUser] = (session.query(AggregateUser).order_by( AggregateUser.user_id).all()) assert len(results) == 0 _update_aggregate_user(session) results: List[AggregateUser] = (session.query(AggregateUser).order_by( AggregateUser.user_id).all()) assert len(results) == 0 prev_id_checkpoint = get_last_indexed_checkpoint( session, AGGREGATE_USER) assert prev_id_checkpoint == 3
def test_index_aggregate_user_populate(app): """Test that we should populate users from empty""" with app.app_context(): db = get_db() with db.scoped_session() as session: results: List[AggregateUser] = (session.query(AggregateUser).order_by( AggregateUser.user_id).all()) assert (len(results) == 0 ), "Test aggregate_user is empty before populate_mock_db()" # create db entries based on entities populate_mock_db(db, basic_entities, block_offset=3) with db.scoped_session() as session: # confirm nothing exists before _update_aggregate_user() results: List[AggregateUser] = (session.query(AggregateUser).order_by( AggregateUser.user_id).all()) assert ( len(results) == 0 ), "Test aggregate_user is empty before _update_aggregate_user()" # trigger celery task _update_aggregate_user(session) # run basic tests against basic_entities basic_tests(session)
def test_es_indexer_catchup(app): with app.app_context(): db = get_db() populate_mock_db(db, basic_entities) try: output = subprocess.run( ["npm", "run", "dev"], env=os.environ, capture_output=True, text=True, cwd="es-indexer", timeout=5, ) raise Exception( f"Elasticsearch indexing stopped: {output.stderr}. With env: {os.environ}" ) except subprocess.TimeoutExpired as timeout: if "catchup done" not in timeout.output.decode("utf-8"): raise Exception("Elasticsearch failed to index") esclient.indices.refresh(index="*") search_res = esclient.search(index="*", query={"match_all": {}})["hits"]["hits"] assert len(search_res) == 6
def test_get_related_artists_too_few_followers(app): """Tests that artists with too few followers get an empty list""" with app.app_context(): db = get_db() populate_mock_db(db, entities) artists = get_related_artists(1, None) assert len(artists) == 0
def test_index_aggregate_monthly_plays_same_checkpoint(app): """Test that we should not update when last index is the same""" # setup with app.app_context(): db = get_db() # run entities = { "tracks": [ {"track_id": 1, "title": "track 1"}, {"track_id": 2, "title": "track 2"}, {"track_id": 3, "title": "track 3"}, {"track_id": 4, "title": "track 4"}, ], "aggregate_monthly_plays": [ { "play_item_id": 3, "timestamp": LAST_YEAR_TIMESTAMP.replace(day=1), "count": 2, }, { "play_item_id": 2, "timestamp": LAST_MONTH_TIMESTAMP.replace(day=1), "count": 1, }, ], "indexing_checkpoints": [ { "tablename": "aggregate_monthly_plays", "last_checkpoint": 9, } ], "plays": [ # Current Plays {"id": 9}, ], } populate_mock_db(db, entities) with db.scoped_session() as session: _index_aggregate_monthly_plays(session) results: List[AggregateMonthlyPlays] = ( session.query(AggregateMonthlyPlays) .order_by(AggregateMonthlyPlays.play_item_id) .all() ) assert len(results) == 2 new_checkpoint: IndexingCheckpoints = ( session.query(IndexingCheckpoints.last_checkpoint) .filter(IndexingCheckpoints.tablename == AGGREGATE_MONTHLY_PLAYS_TABLE_NAME) .scalar() ) assert new_checkpoint == 9
def test_index_aggregate_track_empty_tracks(app): """Test that track metadata without tracks table won't break""" with app.app_context(): db = get_db() entities = { "users": [], "indexing_checkpoints": [{ "tablename": AGGREGATE_TRACK, "last_checkpoint": 0 }], "tracks": [], "reposts": [ { "repost_item_id": 1, "repost_type": "track", "track_id": 1 }, { "repost_item_id": 1, "repost_type": "playlist", "track_id": 1 }, ], "saves": [ { "save_item_id": 1, "save_type": "track", "track_id": 1 }, { "save_item_id": 1, "save_type": "playlist", "track_id": 1 }, ], } populate_mock_db(db, entities) with db.scoped_session() as session: _update_aggregate_track(session) results: List[AggregateTrack] = ( session.query(AggregateTrack).order_by( AggregateTrack.track_id).all()) assert (len(results) == 0 ), "Test that without Tracks there will be no AggregateTracks" prev_id_checkpoint = get_last_indexed_checkpoint( session, AGGREGATE_TRACK) assert prev_id_checkpoint == 1
def test_index_aggregate_track_update(app): """Test that the aggregate_track data is continously added to""" with app.app_context(): db = get_db() entities = deepcopy(basic_entities) entities.update({ "aggregate_track": [ { "track_id": 1, "repost_count": 9, "save_count": 9, }, { "track_id": 2, "repost_count": 9, "save_count": 9, }, { "track_id": 4, "repost_count": 9, "save_count": 9, }, { "track_id": 5, "repost_count": 9, "save_count": 9, }, ], }) populate_mock_db(db, entities, block_offset=3) with db.scoped_session() as session: results: List[AggregateTrack] = ( session.query(AggregateTrack).order_by( AggregateTrack.track_id).all()) assert len(results) == 4, "Test that the entities were created" for i, n in enumerate((0, 1, 3, 4)): assert results[ i].track_id == n + 1, "Test that the entities were created" assert results[ i].repost_count == 9, "Test that the entities were created" assert results[ i].save_count == 9, "Test that the entities were created" _update_aggregate_track(session) with db.scoped_session() as session: basic_tests(session)
def test_index_aggregate_plays_no_plays(app): """Tests that aggregate_plays should skip indexing if there are no plays""" # setup with app.app_context(): db = get_db() # run entities = {"plays": []} populate_mock_db(db, entities) with db.scoped_session() as session: _update_aggregate_plays(session)
def test_get_remixable_tracks(app): with app.app_context(): db = get_db() populate_tracks(db) populate_mock_db( db, { "remixes": [ { "parent_track_id": 9, "child_track_id": 1 }, { "parent_track_id": 8, "child_track_id": 1 }, ], "stems": [ { "parent_track_id": 7, "child_track_id": 1 }, { "parent_track_id": 6, "child_track_id": 1 }, # Verify that tracks with deleted stems are not returned { "parent_track_id": 5, "child_track_id": 10 }, ], "saves": [{ "user_id": 4, "save_item_id": 1 }], "reposts": [{ "user_id": 4, "repost_item_id": 1 }], }, ) with db.scoped_session() as session: _update_aggregate_track(session) tracks = get_remixable_tracks({"with_users": True}) assert len(tracks) == 2 assert tracks[0]["user"]
def test_search_user_tags(app): """Tests that search by tags works for users""" with app.app_context(): db = get_db() test_entities = { "tracks": [ {"track_id": 1, "tags": "pop", "owner_id": 1}, {"track_id": 2, "owner_id": 1, "tags": "pop,rock,electric"}, {"track_id": 3, "owner_id": 2}, {"track_id": 4, "owner_id": 2, "tags": "funk,pop"}, {"track_id": 5, "owner_id": 2, "tags": "funk,pop"}, {"track_id": 6, "owner_id": 2, "tags": "funk,Funk,kpop"}, {"track_id": 7, "owner_id": 3, "tags": "pop"}, {"track_id": 8, "owner_id": 3, "tags": "kpop"}, ], "users": [ {"user_id": 1, "handle": "1"}, {"user_id": 2, "handle": "2"}, {"user_id": 3, "handle": "3"}, ], "follows": [ {"follower_user_id": 1, "followee_user_id": 2}, {"follower_user_id": 1, "followee_user_id": 3}, {"follower_user_id": 2, "followee_user_id": 3}, ], } populate_mock_db(db, test_entities) with db.scoped_session() as session: _update_aggregate_user(session) session.execute("REFRESH MATERIALIZED VIEW tag_track_user") args = { "search_str": "pop", "current_user_id": None, "user_tag_count": 2, "limit": 10, "offset": 0, } users = search_user_tags(session, args) assert len(users) == 2 assert users[0]["user_id"] == 2 # Fir. b/c user 2 has 1 follower assert users[1]["user_id"] == 1 # Sec. b/c user 1 has 0 followers
def test_index_aggregate_user_entity_model(app): """Test that aggregate_user will return information when using seeded entities""" with app.app_context(): db = get_db() entities = { "aggregate_user": [ { "user_id": 1, "track_count": 9, "playlist_count": 9, "album_count": 9, "follower_count": 9, "following_count": 9, "repost_count": 9, "track_save_count": 9, }, { "user_id": 2, "track_count": 9, "playlist_count": 9, "album_count": 9, "follower_count": 9, "following_count": 9, "repost_count": 9, "track_save_count": 9, }, { "user_id": 3, "track_count": 9, "playlist_count": 9, "album_count": 9, "follower_count": 9, "following_count": 9, "repost_count": 9, "track_save_count": 9, }, ], } populate_mock_db(db, entities) with db.scoped_session() as session: results: List[AggregateUser] = (session.query(AggregateUser).order_by( AggregateUser.user_id).all()) created_entity_tests(results, 3)
def test_get_should_update_trending_less_than_hour_no_update(app): last_trending_date = int(datetime(2012, 3, 16, 0, 0).timestamp()) last_block_date = int(datetime(2012, 3, 16, 0, 1).timestamp()) redis_conn = redis.Redis.from_url(url=REDIS_URL) set_last_trending_datetime(redis_conn, last_trending_date) with app.app_context(): db = get_db() web3 = MockWeb3({"timestamp": last_block_date}) # Add some users to the db so we have blocks entities = { "users": [{}] * 3, } populate_mock_db(db, entities) should_update = get_should_update_trending(db, web3, redis_conn, 60 * 60) assert should_update == None
def test_index_aggregate_user_update(app): """Test that the aggregate_user data is overwritten""" with app.app_context(): db = get_db() entities = deepcopy(basic_entities) entities.update({ "aggregate_user": [ { "user_id": 1, "track_count": 9, "playlist_count": 9, "album_count": 9, "follower_count": 9, "following_count": 9, "repost_count": 9, "track_save_count": 9, }, { "user_id": 2, "track_count": 9, "playlist_count": 9, "album_count": 9, "follower_count": 9, "following_count": 9, "repost_count": 9, "track_save_count": 9, }, ], }) populate_mock_db(db, entities, block_offset=3) with db.scoped_session() as session: results: List[AggregateUser] = (session.query(AggregateUser).order_by( AggregateUser.user_id).all()) created_entity_tests(results, 2) _update_aggregate_user(session) with db.scoped_session() as session: basic_tests(session)
def test_es_indexer_processing(app): with app.app_context(): db = get_db() try: proc = subprocess.Popen( ["npm", "run", "dev"], env=os.environ, cwd="es-indexer", text=True, stdout=subprocess.PIPE, ) time.sleep(4) populate_mock_db(db, basic_entities) proc.communicate(timeout=2) except subprocess.TimeoutExpired as timeout: if "processed new updates" not in timeout.output.decode("utf-8"): raise Exception("Elasticsearch failed to process updates") esclient.indices.refresh(index="*") search_res = esclient.search(index="*", query={"match_all": {}})["hits"]["hits"] assert len(search_res) == 6
def test_get_should_update_trending_fifteen_minutes(app): last_trending_date = int(datetime(2012, 3, 16, 0, 0).timestamp()) last_block_date = int(datetime(2012, 3, 16, 0, 16).timestamp()) redis_conn = redis.Redis.from_url(url=REDIS_URL) set_last_trending_datetime(redis_conn, last_trending_date) with app.app_context(): db = get_db() web3 = MockWeb3({"timestamp": last_block_date}) # Add some users to the db so we have blocks entities = { "users": [{}] * 3, } populate_mock_db(db, entities) should_update = get_should_update_trending(db, web3, redis_conn, 60 * 15) assert should_update != None # Result is rounded assert datetime.fromtimestamp(should_update) == datetime( 2012, 3, 16, 0, 15)
async def test_fetch_metadata(app, mocker): with app.app_context(): db = get_db() redis = get_redis() populate_mock_db(db, basic_entities, block_offset=3) solana_client_manager_mock = create_autospec(SolanaClientManager) cid_metadata_client_mock = create_autospec(CIDMetadataClient) cid_metadata_client_mock.async_fetch_metadata_from_gateway_endpoints.return_value = ( mock_cid_metadata ) anchor_program_indexer = AnchorProgramIndexer( PROGRAM_ID, ADMIN_STORAGE_PUBLIC_KEY, LABEL, redis, db, solana_client_manager_mock, cid_metadata_client_mock, ) parsed_tx = { "tx_metadata": { "instructions": [ { "instruction_name": "init_user", "data": Container([("metadata", mock_cid), ("user_id", 1)]), } ] }, "tx_sig": "x4PCuQs3ncvhJ3Qz18CBzYg26KnG1tAD1QvZG9B6oBZbR8cJrat2MzcvCbjtMMn9Mkc4C8w23LHTFaLG4dJaXkV", } mock_parsed_transactions = [parsed_tx] cid_metadata, blacklisted_cids = await anchor_program_indexer.fetch_cid_metadata( mock_parsed_transactions ) assert cid_metadata == mock_cid_metadata
def test_index_aggregate_user_empty_completely(app): """Test a completely empty database won't break""" with app.app_context(): db = get_db() entities = {} populate_mock_db(db, entities, block_offset=3) with db.scoped_session() as session: _update_aggregate_user(session) results: List[AggregateUser] = (session.query(AggregateUser).order_by( AggregateUser.user_id).all()) assert (len(results) == 0 ), "Test that empty entities won't generate AggregateUsers" prev_id_checkpoint = get_last_indexed_checkpoint( session, AGGREGATE_USER) assert prev_id_checkpoint == 0
def test_update_related_artist_scores_if_needed(app): """Tests all cases of update_related_artist_scores_if_needed: not enough followers, existing fresh scores, and needing recalculation""" with app.app_context(): db = get_db() with db.scoped_session() as session: result, _ = update_related_artist_scores_if_needed(session, 0) assert not result, "Don't calculate for low number of followers" populate_mock_db(db, entities) _update_aggregate_user(session) result, _ = update_related_artist_scores_if_needed(session, 0) assert result, "Calculate when followers >= MIN_FOLLOWER_REQUIREMENT (200)" result, _ = update_related_artist_scores_if_needed(session, 0) assert ( not result ), "Don't calculate when scores are already calculated and fresh" session.query(RelatedArtist).update( {RelatedArtist.created_at: datetime.utcnow() - timedelta(weeks=5)}) session.commit() result, _ = update_related_artist_scores_if_needed(session, 0) assert result, "Calculate when the scores are stale"
def test_search_track_tags(app): """Tests that search by tags works fopr tracks""" with app.app_context(): db = get_db() test_entities = { "tracks": [ {"track_id": 1, "tags": "", "owner_id": 1}, {"track_id": 2, "owner_id": 1, "tags": "pop,rock,electric"}, {"track_id": 3, "owner_id": 2}, {"track_id": 4, "owner_id": 2, "tags": "funk,pop"}, {"track_id": 5, "owner_id": 2, "tags": "funk,pop"}, {"track_id": 6, "owner_id": 2, "tags": "funk,Funk,kpop"}, ], "plays": [ {"item_id": 1}, {"item_id": 1}, {"item_id": 2}, {"item_id": 2}, {"item_id": 4}, {"item_id": 5}, {"item_id": 5}, {"item_id": 5}, ], } populate_mock_db(db, test_entities) with db.scoped_session() as session: session.execute("REFRESH MATERIALIZED VIEW tag_track_user") _update_aggregate_plays(session) args = {"search_str": "pop", "current_user_id": None, "limit": 10, "offset": 0} tracks = search_track_tags(session, args) assert len(tracks) == 3 assert tracks[0]["track_id"] == 5 # First w/ 3 plays assert tracks[1]["track_id"] == 2 # Sec w/ 2 plays assert tracks[2]["track_id"] == 4 # Third w/ 1 plays
def test_index_aggregate_track_entity_model(app): """Test that aggregate_track will return information when using seeded entities""" with app.app_context(): db = get_db() entities = { "aggregate_track": [ { "track_id": 1, "repost_count": 9, "save_count": 9, }, { "track_id": 2, "repost_count": 9, "save_count": 9, }, { "track_id": 3, "repost_count": 9, "save_count": 9, }, { "track_id": 4, "repost_count": 9, "save_count": 9, }, ], } populate_mock_db(db, entities) with db.scoped_session() as session: results: List[AggregateTrack] = ( session.query(AggregateTrack).order_by( AggregateTrack.track_id).all()) created_entity_tests(results, 4)
def test_get_top_user_track_tags(app): """Tests that top tags for users can be queried""" with app.app_context(): db = get_db() test_entities = { "tracks": [ { "tags": "" }, {}, { "tags": "pop,rock,electric" }, { "tags": "pop,rock" }, { "tags": "funk,pop" }, ] } populate_mock_db(db, test_entities) with db.scoped_session() as session: session.execute("REFRESH MATERIALIZED VIEW tag_track_user") user_1_tags = _get_top_user_track_tags(session, {"user_id": 1}) user_2_tags = _get_top_user_track_tags(session, {"user_id": 2}) assert len(user_1_tags) == 4 assert user_1_tags[0] == "pop" assert user_1_tags[1] == "rock" assert "electric" in user_1_tags assert "funk" in user_1_tags assert not user_2_tags
def test_prune_plays_skip_prune(app): """Test that we should not prune if there are no plays before cutoff""" # setup with app.app_context(): db = get_db() # run entities = { "tracks": [ { "track_id": 3, "title": "track 3" }, ], "plays": [ # Current Plays { "item_id": 3, "created_at": CURRENT_TIMESTAMP }, ], } populate_mock_db(db, entities) with db.scoped_session() as session: _prune_plays(session, CURRENT_TIMESTAMP) # verify plays plays_result: List[Play] = session.query(Play).order_by(Play.id).all() assert len(plays_result) == 1 # verify archive plays_archive_result: List[PlaysArchive] = ( session.query(PlaysArchive).order_by(PlaysArchive.id).all()) assert len(plays_archive_result) == 0
def setup_trending(db): # Test data # test tracks # when creating tracks, track_id == index test_entities = { "users": [ *[ { "user_id": i + 1, "handle": str(i + 1), "wallet": str(i + 1), # Legacy users have profile_picture and cover_photo (versus _sizes) "profile_picture": "Qm0123456789abcdef0123456789abcdef0123456789ab", "cover_photo": "Qm0123456789abcdef0123456789abcdef0123456789ab", "bio": "filled in", } for i in range(5) ], *[{ "user_id": i + 1, "handle": str(i + 1), "wallet": str(i + 1), "profile_picture_sizes": "Qm0123456789abcdef0123456789abcdef0123456789ab", "cover_photo_sizes": "Qm0123456789abcdef0123456789abcdef0123456789ab", "bio": "filled in", } for i in range(5, 20)], ], "tracks": [ { "track_id": 1, "owner_id": 1 }, { "track_id": 2, "owner_id": 1, "created_at": datetime.now() - timedelta(days=1), }, { "track_id": 3, "owner_id": 2, "created_at": datetime.now() - timedelta(weeks=2), }, { "track_id": 4, "owner_id": 2, "created_at": datetime.now() - timedelta(weeks=6), }, { "track_id": 5, "owner_id": 2, "created_at": datetime.now() - timedelta(weeks=60), }, { "track_id": 6, "owner_id": 2 }, { "track_id": 7, "owner_id": 3 }, { "track_id": 8, "owner_id": 3, "is_delete": True }, { "track_id": 9, "owner_id": 3, "is_unlisted": True }, ], "follows": [ # at least 200 followers for user_0 *[{ "follower_user_id": 3 + i, "followee_user_id": 1 } for i in range(10)], *[{ "follower_user_id": 3 + i, "followee_user_id": 2 } for i in range(15)], *[{ "follower_user_id": 3 + i, "followee_user_id": 3 } for i in range(2)], # Less than 3 followers, so 0 trending score ], "plays": [ *[{ "item_id": 1, "owner_id": 1 } for i in range(10)], *[{ "item_id": 2, "owner_id": 1 } for i in range(12)], *[{ "item_id": 3, "owner_id": 2 } for i in range(13)], *[{ "item_id": 4, "owner_id": 2 } for i in range(14)], *[{ "item_id": 5, "owner_id": 2 } for i in range(15)], *[{ "item_id": 6, "owner_id": 2 } for i in range(16)], *[{ "item_id": 7, "owner_id": 3 } for i in range(17)], *[{ "item_id": 1, "created_at": datetime.now() - timedelta(weeks=3) } for i in range(10)], *[{ "item_id": 1, "created_at": datetime.now() - timedelta(weeks=50) } for i in range(10)], *[{ "item_id": 1, "created_at": datetime.now() - timedelta(weeks=80) } for i in range(10)], *[{ "item_id": 2, "created_at": datetime.now() - timedelta(weeks=2) } for i in range(10)], *[{ "item_id": 3, "created_at": datetime.now() - timedelta(weeks=2) } for i in range(10)], *[{ "item_id": 4, "created_at": datetime.now() - timedelta(weeks=4) } for i in range(10)], *[{ "item_id": 5, "created_at": datetime.now() - timedelta(weeks=5) } for i in range(10)], *[{ "item_id": 6, "created_at": datetime.now() - timedelta(weeks=6) } for i in range(10)], ], "reposts": [ *[{ "repost_item_id": 1, "user_id": i + 1 } for i in range(13)], *[{ "repost_item_id": 1, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=2), } for i in range(20)], *[{ "repost_item_id": 1, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=30), } for i in range(30)], *[{ "repost_item_id": 2, "user_id": i + 1 } for i in range(24)], *[{ "repost_item_id": 3, "user_id": i + 1 } for i in range(25)], *[{ "repost_item_id": 4, "user_id": i + 1 } for i in range(26)], *[{ "repost_item_id": 5, "user_id": i + 1 } for i in range(27)], *[{ "repost_item_id": 6, "user_id": i + 1 } for i in range(28)], *[{ "repost_item_id": 7, "user_id": i + 1 } for i in range(29)], *[{ "repost_item_id": 2, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=2), } for i in range(23)], *[{ "repost_item_id": 3, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=2), } for i in range(23)], *[{ "repost_item_id": 4, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=4), } for i in range(23)], *[{ "repost_item_id": 5, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=5), } for i in range(23)], *[{ "repost_item_id": 6, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=6), } for i in range(23)], ], "saves": [ *[{ "save_item_id": 1, "user_id": i + 1 } for i in range(4)], *[{ "save_item_id": 1, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=3), } for i in range(8)], *[{ "save_item_id": 1, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=50), } for i in range(16)], *[{ "save_item_id": 1, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=60), } for i in range(1)], *[{ "save_item_id": 2, "user_id": i + 1 } for i in range(44)], *[{ "save_item_id": 3, "user_id": i + 1 } for i in range(44)], *[{ "save_item_id": 4, "user_id": i + 1 } for i in range(44)], *[{ "save_item_id": 5, "user_id": i + 1 } for i in range(44)], *[{ "save_item_id": 6, "user_id": i + 1 } for i in range(44)], *[{ "save_item_id": 7, "user_id": i + 1 } for i in range(44)], *[{ "save_item_id": 2, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=2), } for i in range(44)], *[{ "save_item_id": 3, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=2), } for i in range(44)], *[{ "save_item_id": 4, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=4), } for i in range(44)], *[{ "save_item_id": 5, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=5), } for i in range(44)], *[{ "save_item_id": 6, "user_id": i + 1, "created_at": datetime.now() - timedelta(weeks=6), } for i in range(44)], ], } populate_mock_db(db, test_entities)
def test_get_repost_feed_for_user_limit_bounds(app): """ Tests that a repost feed for a user can be queried and respect a limit with deleted tracks. """ with app.app_context(): db = get_db() test_entities = { "reposts": [ # Note these reposts are in chronological order in addition # so the repost feed should pull them "backwards" for reverse chronological # sort order. { "user_id": 1, "repost_item_id": 5, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 2, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 3, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 1, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 4, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 4, "repost_type": "playlist" }, { "user_id": 1, "repost_item_id": 8, "repost_type": "album" }, { "user_id": 1, "repost_item_id": 6, "repost_type": "track" }, ], "tracks": [ { "track_id": 1, "title": "track 1", "is_delete": True }, { "track_id": 2, "title": "track 2" }, { "track_id": 3, "title": "track 3" }, { "track_id": 4, "title": "track 4" }, { "track_id": 5, "title": "track 5" }, { "track_id": 6, "title": "track 6" }, { "track_id": 7, "title": "track 7" }, { "track_id": 8, "title": "track 8" }, ], "playlists": [ { "playlist_id": 1, "playlist_name": "playlist 1" }, { "playlist_id": 2, "playlist_name": "playlist 2" }, { "playlist_id": 3, "playlist_name": "playlist 3" }, { "playlist_id": 4, "playlist_name": "playlist 4" }, { "playlist_id": 5, "playlist_name": "playlist 5" }, { "playlist_id": 6, "playlist_name": "playlist 6" }, { "playlist_id": 7, "playlist_name": "playlist 7" }, { "playlist_id": 8, "playlist_name": "album 8" }, ], } populate_mock_db(db, test_entities) with db.scoped_session() as session: repost_feed = _get_repost_feed_for_user(session, 1, { "limit": 5, "offset": 0 }) # Query for 5 reposts. The problem is the 5th one was deleted, so # we only return 4 here. This is broken. # TODO fix me. assert repost_feed[0]["title"] == "track 6" assert repost_feed[1]["playlist_name"] == "album 8" assert repost_feed[2]["playlist_name"] == "playlist 4" assert repost_feed[3]["title"] == "track 4" # Should skip track 1 because it is deleted assert repost_feed[4]["title"] == "track 3"
def test_prune_plays_old_date(app): """Test that we should archive plays with old dates""" # setup with app.app_context(): db = get_db() # run entities = { "tracks": [ { "track_id": 1, "title": "track 1" }, { "track_id": 2, "title": "track 2" }, { "track_id": 3, "title": "track 3" }, ], "plays": [ # Current Plays { "item_id": 1, "created_at": CURRENT_TIMESTAMP - timedelta(weeks=140) }, { "item_id": 3, "created_at": CURRENT_TIMESTAMP - timedelta(weeks=110) }, { "item_id": 2, "created_at": CURRENT_TIMESTAMP - timedelta(weeks=110) }, { "item_id": 2, "created_at": CURRENT_TIMESTAMP - timedelta(weeks=90) }, { "item_id": 1, "created_at": CURRENT_TIMESTAMP - timedelta(weeks=30) }, { "item_id": 3, "created_at": CURRENT_TIMESTAMP - timedelta(weeks=3) }, { "item_id": 3, "created_at": CURRENT_TIMESTAMP }, ], } populate_mock_db(db, entities) with db.scoped_session() as session: _prune_plays( session, CURRENT_TIMESTAMP, cutoff_timestamp=datetime.now() - timedelta(weeks=6), ) # verify plays plays_result: List[Play] = session.query(Play).order_by(Play.id).all() assert len(plays_result) == 2 assert plays_result[0].id == 6 assert plays_result[0].play_item_id == 3 assert plays_result[0].created_at == CURRENT_TIMESTAMP - timedelta( weeks=3) assert plays_result[1].id == 7 assert plays_result[1].play_item_id == 3 assert plays_result[1].created_at == CURRENT_TIMESTAMP # verify archive plays_archive_result: List[PlaysArchive] = ( session.query(PlaysArchive).order_by(PlaysArchive.id).all()) assert len(plays_archive_result) == 5 assert plays_archive_result[0].id == 1 assert plays_archive_result[0].play_item_id == 1 assert plays_archive_result[0].archived_at == CURRENT_TIMESTAMP assert plays_archive_result[1].id == 2 assert plays_archive_result[1].play_item_id == 3 assert plays_archive_result[1].archived_at == CURRENT_TIMESTAMP assert plays_archive_result[2].id == 3 assert plays_archive_result[2].play_item_id == 2 assert plays_archive_result[2].archived_at == CURRENT_TIMESTAMP assert plays_archive_result[3].id == 4 assert plays_archive_result[3].play_item_id == 2 assert plays_archive_result[3].archived_at == CURRENT_TIMESTAMP assert plays_archive_result[4].id == 5 assert plays_archive_result[4].play_item_id == 1 assert plays_archive_result[4].archived_at == CURRENT_TIMESTAMP
def test_index_aggregate_plays_same_checkpoint(app): """Test that we should not update when last index is the same""" # setup with app.app_context(): db = get_db() # run entities = { "tracks": [ { "track_id": 1, "title": "track 1" }, { "track_id": 2, "title": "track 2" }, { "track_id": 3, "title": "track 3" }, { "track_id": 4, "title": "track 4" }, ], "aggregate_plays": [ # Current Plays { "play_item_id": 1, "count": 3 }, { "play_item_id": 2, "count": 3 }, { "play_item_id": 3, "count": 3 }, ], "indexing_checkpoints": [{ "tablename": "aggregate_plays", "last_checkpoint": 9 }], "plays": [ # Current Plays { "item_id": 1 }, { "item_id": 1 }, { "item_id": 1 }, { "item_id": 2 }, { "item_id": 2 }, { "item_id": 2 }, { "item_id": 3 }, { "item_id": 3 }, { "item_id": 3 }, ], } populate_mock_db(db, entities) with db.scoped_session() as session: _update_aggregate_plays(session) results: List[AggregatePlays] = ( session.query(AggregatePlays).order_by( AggregatePlays.play_item_id).all()) assert len(results) == 3
def test_index_aggregate_plays_populate(app): """Test that we should populate plays from empty""" date = datetime.now() # setup with app.app_context(): db = get_db() # run entities = { "tracks": [ { "track_id": 0, "title": "track 0" }, { "track_id": 1, "title": "track 1" }, { "track_id": 2, "title": "track 2" }, { "track_id": 3, "title": "track 3" }, { "track_id": 4, "title": "track 4" }, ], "plays": [ # Current Plays { "item_id": 0 }, { "item_id": 0 }, { "item_id": 1 }, { "item_id": 1 }, { "item_id": 2 }, { "item_id": 3 }, # > 1 wk plays { "item_id": 2, "created_at": date - timedelta(weeks=2) }, { "item_id": 2, "created_at": date - timedelta(weeks=2) }, { "item_id": 3, "created_at": date - timedelta(weeks=2) }, { "item_id": 3 }, { "item_id": 3 }, { "item_id": 4 }, { "item_id": 4 }, ], } populate_mock_db(db, entities) with db.scoped_session() as session: _update_aggregate_plays(session) results: List[AggregatePlays] = ( session.query(AggregatePlays).order_by( AggregatePlays.play_item_id).all()) assert len(results) == 5 assert results[0].play_item_id == 0 assert results[0].count == 2 assert results[1].play_item_id == 1 assert results[1].count == 2 assert results[2].play_item_id == 2 assert results[2].count == 3 assert results[3].play_item_id == 3 assert results[3].count == 4 assert results[4].play_item_id == 4 assert results[4].count == 2
def test_calculate_related_artists_scores(app): with app.app_context(): db = get_db() populate_mock_db(db, entities) with db.scoped_session() as session: _update_aggregate_user(session) # Check sampled (with large enough sample to get all rows for deterministic result) rows = _calculate_related_artists_scores( session, 0, sample_size=200 + 50 + 100 + 40 + 5 + 500 + 200, # sum of all the follows ) assert rows[0].related_artist_user_id == 1 and math.isclose( rows[0].score, 50, abs_tol=0.001) assert rows[1].related_artist_user_id == 2 and math.isclose( rows[1].score, 25, abs_tol=0.001) assert rows[2].related_artist_user_id == 6 and math.isclose( rows[2].score, 18, abs_tol=0.001) assert rows[3].related_artist_user_id == 3 and math.isclose( rows[3].score, 10, abs_tol=0.001) assert rows[4].related_artist_user_id == 5 and math.isclose( rows[4].score, 5, abs_tol=0.001) assert rows[5].related_artist_user_id == 4 and math.isclose( rows[5].score, 3.2, abs_tol=0.001) # Check unsampled rows = _calculate_related_artists_scores(session, 0) assert rows[0].related_artist_user_id == 1 and math.isclose( rows[0].score, 50, abs_tol=0.001) assert rows[1].related_artist_user_id == 2 and math.isclose( rows[1].score, 25, abs_tol=0.001) assert rows[2].related_artist_user_id == 6 and math.isclose( rows[2].score, 18, abs_tol=0.001) assert rows[3].related_artist_user_id == 3 and math.isclose( rows[3].score, 10, abs_tol=0.001) assert rows[4].related_artist_user_id == 5 and math.isclose( rows[4].score, 5, abs_tol=0.001) assert rows[5].related_artist_user_id == 4 and math.isclose( rows[5].score, 3.2, abs_tol=0.001) # Check edge case with 0 followers populate_mock_db( db, {"follows": [{ "follower_user_id": 100, "followee_user_id": 7 }]}) rows = _calculate_related_artists_scores(session, 0) # Same results as unsampled. Shouldn't throw DivideByZero exception assert rows[0].related_artist_user_id == 1 and math.isclose( rows[0].score, 50, abs_tol=0.001) assert rows[1].related_artist_user_id == 2 and math.isclose( rows[1].score, 25, abs_tol=0.001) assert rows[2].related_artist_user_id == 6 and math.isclose( rows[2].score, 18, abs_tol=0.001) assert rows[3].related_artist_user_id == 3 and math.isclose( rows[3].score, 10, abs_tol=0.001) assert rows[4].related_artist_user_id == 5 and math.isclose( rows[4].score, 5, abs_tol=0.001) assert rows[5].related_artist_user_id == 4 and math.isclose( rows[5].score, 3.2, abs_tol=0.001)
def test_get_repost_feed_for_user(app): """Tests that a repost feed for a user can be queried""" with app.app_context(): db = get_db() test_entities = { "reposts": [ # Note these reposts are in chronological order in addition # so the repost feed should pull them "backwards" for reverse chronological # sort order. { "user_id": 1, "repost_item_id": 5, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 2, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 3, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 1, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 4, "repost_type": "track" }, { "user_id": 1, "repost_item_id": 4, "repost_type": "playlist" }, { "user_id": 1, "repost_item_id": 8, "repost_type": "album" }, { "user_id": 1, "repost_item_id": 6, "repost_type": "track" }, ], "tracks": [ { "track_id": 1, "title": "track 1" }, { "track_id": 2, "title": "track 2" }, { "track_id": 3, "title": "track 3" }, { "track_id": 4, "title": "track 4" }, { "track_id": 5, "title": "track 5" }, { "track_id": 6, "title": "track 6" }, { "track_id": 7, "title": "track 7" }, { "track_id": 8, "title": "track 8" }, ], "playlists": [ { "playlist_id": 1, "playlist_name": "playlist 1" }, { "playlist_id": 2, "playlist_name": "playlist 2" }, { "playlist_id": 3, "playlist_name": "playlist 3" }, { "playlist_id": 4, "playlist_name": "playlist 4" }, { "playlist_id": 5, "playlist_name": "playlist 5" }, { "playlist_id": 6, "playlist_name": "playlist 6" }, { "playlist_id": 7, "playlist_name": "playlist 7" }, { "playlist_id": 8, "playlist_name": "album 8" }, ], } populate_mock_db(db, test_entities) with db.scoped_session() as session: repost_feed = _get_repost_feed_for_user(session, 1, { "limit": 10, "offset": 0 }) assert repost_feed[0]["title"] == "track 6" assert repost_feed[1]["playlist_name"] == "album 8" assert repost_feed[2]["playlist_name"] == "playlist 4" assert repost_feed[3]["title"] == "track 4" assert repost_feed[4]["title"] == "track 1" assert repost_feed[5]["title"] == "track 3" assert repost_feed[6]["title"] == "track 2" assert repost_feed[7]["title"] == "track 5"
def test_fetch_and_parse_sol_rewards_transfer_instruction(app): # pylint: disable=W0621 with app.app_context(): db = get_db() redis = get_redis() solana_client_manager_mock = create_autospec(SolanaClientManager) solana_client_manager_mock.get_sol_tx_info.return_value = mock_tx_info first_tx_sig = "tx_sig_one" second_tx_sig = "tx_sig_two" parsed_tx = fetch_and_parse_sol_rewards_transfer_instruction( solana_client_manager_mock, first_tx_sig) assert (parsed_tx["transfer_instruction"]["amount"] # pylint: disable=E1136 == 10000000000) assert (parsed_tx["transfer_instruction"]["eth_recipient"] # pylint: disable=E1136 == "0x0403be3560116a12b467855cb29a393174a59876") assert (parsed_tx["transfer_instruction"]["challenge_id"] # pylint: disable=E1136 == "profile-completion") assert parsed_tx["tx_sig"] == first_tx_sig assert parsed_tx["slot"] == 72131741 test_user_entries = { "users": [ { "user_id": 1, "handle": "piazza", "wallet": "0x0403be3560116a12b467855cb29a393174a59876", }, ] } with db.scoped_session() as session: process_batch_sol_reward_manager_txs(session, [parsed_tx], redis) disbursments = session.query(ChallengeDisbursement).all() assert len(disbursments) == 1 disbursement = disbursments[0] # Assert that this invalid user was set to user_id 0 assert disbursement.user_id == 0 reward_manager_tx_1 = (session.query(RewardManagerTransaction).filter( RewardManagerTransaction.signature == first_tx_sig).all()) assert len(reward_manager_tx_1) == 1 populate_mock_db(db, test_user_entries) parsed_tx["tx_sig"] = second_tx_sig next_slot = parsed_tx["slot"] + 1 parsed_tx["slot"] = next_slot parsed_tx["transfer_instruction"]["challenge_id"] = "tt" with db.scoped_session() as session: process_batch_sol_reward_manager_txs(session, [parsed_tx], redis) disbursments = (session.query(ChallengeDisbursement).order_by( desc(ChallengeDisbursement.slot)).all()) reward_manager_tx_1 = (session.query(RewardManagerTransaction).filter( RewardManagerTransaction.signature == second_tx_sig).all()) assert len(reward_manager_tx_1) == 1 assert len(disbursments) == 2 disbursment = disbursments[0] assert disbursment.challenge_id == "tt" assert disbursment.user_id == 1 assert disbursment.signature == second_tx_sig assert disbursment.slot == next_slot assert disbursment.specifier == "123456789" reward_manager_tx_2 = (session.query(RewardManagerTransaction).filter( RewardManagerTransaction.signature == second_tx_sig).all()) assert len(reward_manager_tx_2) == 1