def test_index_aggregate_user_same_checkpoint(app):
"""Test that we should not update when last index is the same"""
with app.app_context():
db = get_db()
entities = deepcopy(basic_entities)
current_blocknumber = basic_entities["blocks"][0]["number"]
entities.update({
"indexing_checkpoints": [{
"tablename": AGGREGATE_USER,
"last_checkpoint": current_blocknumber
}],
})
populate_mock_db(db, entities)
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert len(results) == 0
_update_aggregate_user(session)
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert len(results) == 0
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 3
def test_index_aggregate_user_populate(app):
"""Test that we should populate users from empty"""
with app.app_context():
db = get_db()
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (len(results) == 0
), "Test aggregate_user is empty before populate_mock_db()"
# create db entries based on entities
populate_mock_db(db, basic_entities, block_offset=3)
with db.scoped_session() as session:
# confirm nothing exists before _update_aggregate_user()
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (
len(results) == 0
), "Test aggregate_user is empty before _update_aggregate_user()"
# trigger celery task
_update_aggregate_user(session)
# run basic tests against basic_entities
basic_tests(session)
def test_es_indexer_catchup(app):
with app.app_context():
db = get_db()
populate_mock_db(db, basic_entities)
try:
output = subprocess.run(
["npm", "run", "dev"],
env=os.environ,
capture_output=True,
text=True,
cwd="es-indexer",
timeout=5,
)
raise Exception(
f"Elasticsearch indexing stopped: {output.stderr}. With env: {os.environ}"
)
except subprocess.TimeoutExpired as timeout:
if "catchup done" not in timeout.output.decode("utf-8"):
raise Exception("Elasticsearch failed to index")
esclient.indices.refresh(index="*")
search_res = esclient.search(index="*", query={"match_all":
{}})["hits"]["hits"]
assert len(search_res) == 6
def test_get_related_artists_too_few_followers(app):
"""Tests that artists with too few followers get an empty list"""
with app.app_context():
db = get_db()
populate_mock_db(db, entities)
artists = get_related_artists(1, None)
assert len(artists) == 0
def test_index_aggregate_monthly_plays_same_checkpoint(app):
"""Test that we should not update when last index is the same"""
# setup
with app.app_context():
db = get_db()
# run
entities = {
"tracks": [
{"track_id": 1, "title": "track 1"},
{"track_id": 2, "title": "track 2"},
{"track_id": 3, "title": "track 3"},
{"track_id": 4, "title": "track 4"},
],
"aggregate_monthly_plays": [
{
"play_item_id": 3,
"timestamp": LAST_YEAR_TIMESTAMP.replace(day=1),
"count": 2,
},
{
"play_item_id": 2,
"timestamp": LAST_MONTH_TIMESTAMP.replace(day=1),
"count": 1,
},
],
"indexing_checkpoints": [
{
"tablename": "aggregate_monthly_plays",
"last_checkpoint": 9,
}
],
"plays": [
# Current Plays
{"id": 9},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_index_aggregate_monthly_plays(session)
results: List[AggregateMonthlyPlays] = (
session.query(AggregateMonthlyPlays)
.order_by(AggregateMonthlyPlays.play_item_id)
.all()
)
assert len(results) == 2
new_checkpoint: IndexingCheckpoints = (
session.query(IndexingCheckpoints.last_checkpoint)
.filter(IndexingCheckpoints.tablename == AGGREGATE_MONTHLY_PLAYS_TABLE_NAME)
.scalar()
)
assert new_checkpoint == 9
def test_index_aggregate_track_empty_tracks(app):
"""Test that track metadata without tracks table won't break"""
with app.app_context():
db = get_db()
entities = {
"users": [],
"indexing_checkpoints": [{
"tablename": AGGREGATE_TRACK,
"last_checkpoint": 0
}],
"tracks": [],
"reposts": [
{
"repost_item_id": 1,
"repost_type": "track",
"track_id": 1
},
{
"repost_item_id": 1,
"repost_type": "playlist",
"track_id": 1
},
],
"saves": [
{
"save_item_id": 1,
"save_type": "track",
"track_id": 1
},
{
"save_item_id": 1,
"save_type": "playlist",
"track_id": 1
},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_update_aggregate_track(session)
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert (len(results) == 0
), "Test that without Tracks there will be no AggregateTracks"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_TRACK)
assert prev_id_checkpoint == 1
def test_index_aggregate_track_update(app):
"""Test that the aggregate_track data is continously added to"""
with app.app_context():
db = get_db()
entities = deepcopy(basic_entities)
entities.update({
"aggregate_track": [
{
"track_id": 1,
"repost_count": 9,
"save_count": 9,
},
{
"track_id": 2,
"repost_count": 9,
"save_count": 9,
},
{
"track_id": 4,
"repost_count": 9,
"save_count": 9,
},
{
"track_id": 5,
"repost_count": 9,
"save_count": 9,
},
],
})
populate_mock_db(db, entities, block_offset=3)
with db.scoped_session() as session:
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert len(results) == 4, "Test that the entities were created"
for i, n in enumerate((0, 1, 3, 4)):
assert results[
i].track_id == n + 1, "Test that the entities were created"
assert results[
i].repost_count == 9, "Test that the entities were created"
assert results[
i].save_count == 9, "Test that the entities were created"
_update_aggregate_track(session)
with db.scoped_session() as session:
basic_tests(session)
def test_index_aggregate_plays_no_plays(app):
"""Tests that aggregate_plays should skip indexing if there are no plays"""
# setup
with app.app_context():
db = get_db()
# run
entities = {"plays": []}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_update_aggregate_plays(session)
def test_get_remixable_tracks(app):
with app.app_context():
db = get_db()
populate_tracks(db)
populate_mock_db(
db,
{
"remixes": [
{
"parent_track_id": 9,
"child_track_id": 1
},
{
"parent_track_id": 8,
"child_track_id": 1
},
],
"stems": [
{
"parent_track_id": 7,
"child_track_id": 1
},
{
"parent_track_id": 6,
"child_track_id": 1
},
# Verify that tracks with deleted stems are not returned
{
"parent_track_id": 5,
"child_track_id": 10
},
],
"saves": [{
"user_id": 4,
"save_item_id": 1
}],
"reposts": [{
"user_id": 4,
"repost_item_id": 1
}],
},
)
with db.scoped_session() as session:
_update_aggregate_track(session)
tracks = get_remixable_tracks({"with_users": True})
assert len(tracks) == 2
assert tracks[0]["user"]
def test_search_user_tags(app):
"""Tests that search by tags works for users"""
with app.app_context():
db = get_db()
test_entities = {
"tracks": [
{"track_id": 1, "tags": "pop", "owner_id": 1},
{"track_id": 2, "owner_id": 1, "tags": "pop,rock,electric"},
{"track_id": 3, "owner_id": 2},
{"track_id": 4, "owner_id": 2, "tags": "funk,pop"},
{"track_id": 5, "owner_id": 2, "tags": "funk,pop"},
{"track_id": 6, "owner_id": 2, "tags": "funk,Funk,kpop"},
{"track_id": 7, "owner_id": 3, "tags": "pop"},
{"track_id": 8, "owner_id": 3, "tags": "kpop"},
],
"users": [
{"user_id": 1, "handle": "1"},
{"user_id": 2, "handle": "2"},
{"user_id": 3, "handle": "3"},
],
"follows": [
{"follower_user_id": 1, "followee_user_id": 2},
{"follower_user_id": 1, "followee_user_id": 3},
{"follower_user_id": 2, "followee_user_id": 3},
],
}
populate_mock_db(db, test_entities)
with db.scoped_session() as session:
_update_aggregate_user(session)
session.execute("REFRESH MATERIALIZED VIEW tag_track_user")
args = {
"search_str": "pop",
"current_user_id": None,
"user_tag_count": 2,
"limit": 10,
"offset": 0,
}
users = search_user_tags(session, args)
assert len(users) == 2
assert users[0]["user_id"] == 2 # Fir. b/c user 2 has 1 follower
assert users[1]["user_id"] == 1 # Sec. b/c user 1 has 0 followers
def test_index_aggregate_user_entity_model(app):
"""Test that aggregate_user will return information when using seeded entities"""
with app.app_context():
db = get_db()
entities = {
"aggregate_user": [
{
"user_id": 1,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
{
"user_id": 2,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
{
"user_id": 3,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
created_entity_tests(results, 3)
def test_get_should_update_trending_less_than_hour_no_update(app):
last_trending_date = int(datetime(2012, 3, 16, 0, 0).timestamp())
last_block_date = int(datetime(2012, 3, 16, 0, 1).timestamp())
redis_conn = redis.Redis.from_url(url=REDIS_URL)
set_last_trending_datetime(redis_conn, last_trending_date)
with app.app_context():
db = get_db()
web3 = MockWeb3({"timestamp": last_block_date})
# Add some users to the db so we have blocks
entities = {
"users": [{}] * 3,
}
populate_mock_db(db, entities)
should_update = get_should_update_trending(db, web3, redis_conn, 60 * 60)
assert should_update == None
def test_index_aggregate_user_update(app):
"""Test that the aggregate_user data is overwritten"""
with app.app_context():
db = get_db()
entities = deepcopy(basic_entities)
entities.update({
"aggregate_user": [
{
"user_id": 1,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
{
"user_id": 2,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
],
})
populate_mock_db(db, entities, block_offset=3)
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
created_entity_tests(results, 2)
_update_aggregate_user(session)
with db.scoped_session() as session:
basic_tests(session)
def test_es_indexer_processing(app):
with app.app_context():
db = get_db()
try:
proc = subprocess.Popen(
["npm", "run", "dev"],
env=os.environ,
cwd="es-indexer",
text=True,
stdout=subprocess.PIPE,
)
time.sleep(4)
populate_mock_db(db, basic_entities)
proc.communicate(timeout=2)
except subprocess.TimeoutExpired as timeout:
if "processed new updates" not in timeout.output.decode("utf-8"):
raise Exception("Elasticsearch failed to process updates")
esclient.indices.refresh(index="*")
search_res = esclient.search(index="*", query={"match_all":
{}})["hits"]["hits"]
assert len(search_res) == 6
def test_get_should_update_trending_fifteen_minutes(app):
last_trending_date = int(datetime(2012, 3, 16, 0, 0).timestamp())
last_block_date = int(datetime(2012, 3, 16, 0, 16).timestamp())
redis_conn = redis.Redis.from_url(url=REDIS_URL)
set_last_trending_datetime(redis_conn, last_trending_date)
with app.app_context():
db = get_db()
web3 = MockWeb3({"timestamp": last_block_date})
# Add some users to the db so we have blocks
entities = {
"users": [{}] * 3,
}
populate_mock_db(db, entities)
should_update = get_should_update_trending(db, web3, redis_conn, 60 * 15)
assert should_update != None
# Result is rounded
assert datetime.fromtimestamp(should_update) == datetime(
2012, 3, 16, 0, 15)
async def test_fetch_metadata(app, mocker):
with app.app_context():
db = get_db()
redis = get_redis()
populate_mock_db(db, basic_entities, block_offset=3)
solana_client_manager_mock = create_autospec(SolanaClientManager)
cid_metadata_client_mock = create_autospec(CIDMetadataClient)
cid_metadata_client_mock.async_fetch_metadata_from_gateway_endpoints.return_value = (
mock_cid_metadata
)
anchor_program_indexer = AnchorProgramIndexer(
PROGRAM_ID,
ADMIN_STORAGE_PUBLIC_KEY,
LABEL,
redis,
db,
solana_client_manager_mock,
cid_metadata_client_mock,
)
parsed_tx = {
"tx_metadata": {
"instructions": [
{
"instruction_name": "init_user",
"data": Container([("metadata", mock_cid), ("user_id", 1)]),
}
]
},
"tx_sig": "x4PCuQs3ncvhJ3Qz18CBzYg26KnG1tAD1QvZG9B6oBZbR8cJrat2MzcvCbjtMMn9Mkc4C8w23LHTFaLG4dJaXkV",
}
mock_parsed_transactions = [parsed_tx]
cid_metadata, blacklisted_cids = await anchor_program_indexer.fetch_cid_metadata(
mock_parsed_transactions
)
assert cid_metadata == mock_cid_metadata
def test_index_aggregate_user_empty_completely(app):
"""Test a completely empty database won't break"""
with app.app_context():
db = get_db()
entities = {}
populate_mock_db(db, entities, block_offset=3)
with db.scoped_session() as session:
_update_aggregate_user(session)
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (len(results) == 0
), "Test that empty entities won't generate AggregateUsers"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 0
def test_update_related_artist_scores_if_needed(app):
"""Tests all cases of update_related_artist_scores_if_needed: not enough followers, existing fresh scores,
and needing recalculation"""
with app.app_context():
db = get_db()
with db.scoped_session() as session:
result, _ = update_related_artist_scores_if_needed(session, 0)
assert not result, "Don't calculate for low number of followers"
populate_mock_db(db, entities)
_update_aggregate_user(session)
result, _ = update_related_artist_scores_if_needed(session, 0)
assert result, "Calculate when followers >= MIN_FOLLOWER_REQUIREMENT (200)"
result, _ = update_related_artist_scores_if_needed(session, 0)
assert (
not result
), "Don't calculate when scores are already calculated and fresh"
session.query(RelatedArtist).update(
{RelatedArtist.created_at: datetime.utcnow() - timedelta(weeks=5)})
session.commit()
result, _ = update_related_artist_scores_if_needed(session, 0)
assert result, "Calculate when the scores are stale"
def test_search_track_tags(app):
"""Tests that search by tags works fopr tracks"""
with app.app_context():
db = get_db()
test_entities = {
"tracks": [
{"track_id": 1, "tags": "", "owner_id": 1},
{"track_id": 2, "owner_id": 1, "tags": "pop,rock,electric"},
{"track_id": 3, "owner_id": 2},
{"track_id": 4, "owner_id": 2, "tags": "funk,pop"},
{"track_id": 5, "owner_id": 2, "tags": "funk,pop"},
{"track_id": 6, "owner_id": 2, "tags": "funk,Funk,kpop"},
],
"plays": [
{"item_id": 1},
{"item_id": 1},
{"item_id": 2},
{"item_id": 2},
{"item_id": 4},
{"item_id": 5},
{"item_id": 5},
{"item_id": 5},
],
}
populate_mock_db(db, test_entities)
with db.scoped_session() as session:
session.execute("REFRESH MATERIALIZED VIEW tag_track_user")
_update_aggregate_plays(session)
args = {"search_str": "pop", "current_user_id": None, "limit": 10, "offset": 0}
tracks = search_track_tags(session, args)
assert len(tracks) == 3
assert tracks[0]["track_id"] == 5 # First w/ 3 plays
assert tracks[1]["track_id"] == 2 # Sec w/ 2 plays
assert tracks[2]["track_id"] == 4 # Third w/ 1 plays
def test_index_aggregate_track_entity_model(app):
"""Test that aggregate_track will return information when using seeded entities"""
with app.app_context():
db = get_db()
entities = {
"aggregate_track": [
{
"track_id": 1,
"repost_count": 9,
"save_count": 9,
},
{
"track_id": 2,
"repost_count": 9,
"save_count": 9,
},
{
"track_id": 3,
"repost_count": 9,
"save_count": 9,
},
{
"track_id": 4,
"repost_count": 9,
"save_count": 9,
},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
created_entity_tests(results, 4)
def test_get_top_user_track_tags(app):
"""Tests that top tags for users can be queried"""
with app.app_context():
db = get_db()
test_entities = {
"tracks": [
{
"tags": ""
},
{},
{
"tags": "pop,rock,electric"
},
{
"tags": "pop,rock"
},
{
"tags": "funk,pop"
},
]
}
populate_mock_db(db, test_entities)
with db.scoped_session() as session:
session.execute("REFRESH MATERIALIZED VIEW tag_track_user")
user_1_tags = _get_top_user_track_tags(session, {"user_id": 1})
user_2_tags = _get_top_user_track_tags(session, {"user_id": 2})
assert len(user_1_tags) == 4
assert user_1_tags[0] == "pop"
assert user_1_tags[1] == "rock"
assert "electric" in user_1_tags
assert "funk" in user_1_tags
assert not user_2_tags
def test_prune_plays_skip_prune(app):
"""Test that we should not prune if there are no plays before cutoff"""
# setup
with app.app_context():
db = get_db()
# run
entities = {
"tracks": [
{
"track_id": 3,
"title": "track 3"
},
],
"plays": [
# Current Plays
{
"item_id": 3,
"created_at": CURRENT_TIMESTAMP
},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_prune_plays(session, CURRENT_TIMESTAMP)
# verify plays
plays_result: List[Play] = session.query(Play).order_by(Play.id).all()
assert len(plays_result) == 1
# verify archive
plays_archive_result: List[PlaysArchive] = (
session.query(PlaysArchive).order_by(PlaysArchive.id).all())
assert len(plays_archive_result) == 0
def setup_trending(db):
# Test data
# test tracks
# when creating tracks, track_id == index
test_entities = {
"users": [
*[
{
"user_id": i + 1,
"handle": str(i + 1),
"wallet": str(i + 1),
# Legacy users have profile_picture and cover_photo (versus _sizes)
"profile_picture":
"Qm0123456789abcdef0123456789abcdef0123456789ab",
"cover_photo":
"Qm0123456789abcdef0123456789abcdef0123456789ab",
"bio": "filled in",
} for i in range(5)
],
*[{
"user_id": i + 1,
"handle": str(i + 1),
"wallet": str(i + 1),
"profile_picture_sizes":
"Qm0123456789abcdef0123456789abcdef0123456789ab",
"cover_photo_sizes":
"Qm0123456789abcdef0123456789abcdef0123456789ab",
"bio": "filled in",
} for i in range(5, 20)],
],
"tracks": [
{
"track_id": 1,
"owner_id": 1
},
{
"track_id": 2,
"owner_id": 1,
"created_at": datetime.now() - timedelta(days=1),
},
{
"track_id": 3,
"owner_id": 2,
"created_at": datetime.now() - timedelta(weeks=2),
},
{
"track_id": 4,
"owner_id": 2,
"created_at": datetime.now() - timedelta(weeks=6),
},
{
"track_id": 5,
"owner_id": 2,
"created_at": datetime.now() - timedelta(weeks=60),
},
{
"track_id": 6,
"owner_id": 2
},
{
"track_id": 7,
"owner_id": 3
},
{
"track_id": 8,
"owner_id": 3,
"is_delete": True
},
{
"track_id": 9,
"owner_id": 3,
"is_unlisted": True
},
],
"follows": [
# at least 200 followers for user_0
*[{
"follower_user_id": 3 + i,
"followee_user_id": 1
} for i in range(10)],
*[{
"follower_user_id": 3 + i,
"followee_user_id": 2
} for i in range(15)],
*[{
"follower_user_id": 3 + i,
"followee_user_id": 3
} for i in range(2)], # Less than 3 followers, so 0 trending score
],
"plays": [
*[{
"item_id": 1,
"owner_id": 1
} for i in range(10)],
*[{
"item_id": 2,
"owner_id": 1
} for i in range(12)],
*[{
"item_id": 3,
"owner_id": 2
} for i in range(13)],
*[{
"item_id": 4,
"owner_id": 2
} for i in range(14)],
*[{
"item_id": 5,
"owner_id": 2
} for i in range(15)],
*[{
"item_id": 6,
"owner_id": 2
} for i in range(16)],
*[{
"item_id": 7,
"owner_id": 3
} for i in range(17)],
*[{
"item_id": 1,
"created_at": datetime.now() - timedelta(weeks=3)
} for i in range(10)],
*[{
"item_id": 1,
"created_at": datetime.now() - timedelta(weeks=50)
} for i in range(10)],
*[{
"item_id": 1,
"created_at": datetime.now() - timedelta(weeks=80)
} for i in range(10)],
*[{
"item_id": 2,
"created_at": datetime.now() - timedelta(weeks=2)
} for i in range(10)],
*[{
"item_id": 3,
"created_at": datetime.now() - timedelta(weeks=2)
} for i in range(10)],
*[{
"item_id": 4,
"created_at": datetime.now() - timedelta(weeks=4)
} for i in range(10)],
*[{
"item_id": 5,
"created_at": datetime.now() - timedelta(weeks=5)
} for i in range(10)],
*[{
"item_id": 6,
"created_at": datetime.now() - timedelta(weeks=6)
} for i in range(10)],
],
"reposts": [
*[{
"repost_item_id": 1,
"user_id": i + 1
} for i in range(13)],
*[{
"repost_item_id": 1,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=2),
} for i in range(20)],
*[{
"repost_item_id": 1,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=30),
} for i in range(30)],
*[{
"repost_item_id": 2,
"user_id": i + 1
} for i in range(24)],
*[{
"repost_item_id": 3,
"user_id": i + 1
} for i in range(25)],
*[{
"repost_item_id": 4,
"user_id": i + 1
} for i in range(26)],
*[{
"repost_item_id": 5,
"user_id": i + 1
} for i in range(27)],
*[{
"repost_item_id": 6,
"user_id": i + 1
} for i in range(28)],
*[{
"repost_item_id": 7,
"user_id": i + 1
} for i in range(29)],
*[{
"repost_item_id": 2,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=2),
} for i in range(23)],
*[{
"repost_item_id": 3,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=2),
} for i in range(23)],
*[{
"repost_item_id": 4,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=4),
} for i in range(23)],
*[{
"repost_item_id": 5,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=5),
} for i in range(23)],
*[{
"repost_item_id": 6,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=6),
} for i in range(23)],
],
"saves": [
*[{
"save_item_id": 1,
"user_id": i + 1
} for i in range(4)],
*[{
"save_item_id": 1,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=3),
} for i in range(8)],
*[{
"save_item_id": 1,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=50),
} for i in range(16)],
*[{
"save_item_id": 1,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=60),
} for i in range(1)],
*[{
"save_item_id": 2,
"user_id": i + 1
} for i in range(44)],
*[{
"save_item_id": 3,
"user_id": i + 1
} for i in range(44)],
*[{
"save_item_id": 4,
"user_id": i + 1
} for i in range(44)],
*[{
"save_item_id": 5,
"user_id": i + 1
} for i in range(44)],
*[{
"save_item_id": 6,
"user_id": i + 1
} for i in range(44)],
*[{
"save_item_id": 7,
"user_id": i + 1
} for i in range(44)],
*[{
"save_item_id": 2,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=2),
} for i in range(44)],
*[{
"save_item_id": 3,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=2),
} for i in range(44)],
*[{
"save_item_id": 4,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=4),
} for i in range(44)],
*[{
"save_item_id": 5,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=5),
} for i in range(44)],
*[{
"save_item_id": 6,
"user_id": i + 1,
"created_at": datetime.now() - timedelta(weeks=6),
} for i in range(44)],
],
}
populate_mock_db(db, test_entities)
def test_get_repost_feed_for_user_limit_bounds(app):
"""
Tests that a repost feed for a user can be queried and respect a limit
with deleted tracks.
"""
with app.app_context():
db = get_db()
test_entities = {
"reposts": [
# Note these reposts are in chronological order in addition
# so the repost feed should pull them "backwards" for reverse chronological
# sort order.
{
"user_id": 1,
"repost_item_id": 5,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 2,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 3,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 1,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 4,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 4,
"repost_type": "playlist"
},
{
"user_id": 1,
"repost_item_id": 8,
"repost_type": "album"
},
{
"user_id": 1,
"repost_item_id": 6,
"repost_type": "track"
},
],
"tracks": [
{
"track_id": 1,
"title": "track 1",
"is_delete": True
},
{
"track_id": 2,
"title": "track 2"
},
{
"track_id": 3,
"title": "track 3"
},
{
"track_id": 4,
"title": "track 4"
},
{
"track_id": 5,
"title": "track 5"
},
{
"track_id": 6,
"title": "track 6"
},
{
"track_id": 7,
"title": "track 7"
},
{
"track_id": 8,
"title": "track 8"
},
],
"playlists": [
{
"playlist_id": 1,
"playlist_name": "playlist 1"
},
{
"playlist_id": 2,
"playlist_name": "playlist 2"
},
{
"playlist_id": 3,
"playlist_name": "playlist 3"
},
{
"playlist_id": 4,
"playlist_name": "playlist 4"
},
{
"playlist_id": 5,
"playlist_name": "playlist 5"
},
{
"playlist_id": 6,
"playlist_name": "playlist 6"
},
{
"playlist_id": 7,
"playlist_name": "playlist 7"
},
{
"playlist_id": 8,
"playlist_name": "album 8"
},
],
}
populate_mock_db(db, test_entities)
with db.scoped_session() as session:
repost_feed = _get_repost_feed_for_user(session, 1, {
"limit": 5,
"offset": 0
})
# Query for 5 reposts. The problem is the 5th one was deleted, so
# we only return 4 here. This is broken.
# TODO fix me.
assert repost_feed[0]["title"] == "track 6"
assert repost_feed[1]["playlist_name"] == "album 8"
assert repost_feed[2]["playlist_name"] == "playlist 4"
assert repost_feed[3]["title"] == "track 4"
# Should skip track 1 because it is deleted
assert repost_feed[4]["title"] == "track 3"
def test_prune_plays_old_date(app):
"""Test that we should archive plays with old dates"""
# setup
with app.app_context():
db = get_db()
# run
entities = {
"tracks": [
{
"track_id": 1,
"title": "track 1"
},
{
"track_id": 2,
"title": "track 2"
},
{
"track_id": 3,
"title": "track 3"
},
],
"plays": [
# Current Plays
{
"item_id": 1,
"created_at": CURRENT_TIMESTAMP - timedelta(weeks=140)
},
{
"item_id": 3,
"created_at": CURRENT_TIMESTAMP - timedelta(weeks=110)
},
{
"item_id": 2,
"created_at": CURRENT_TIMESTAMP - timedelta(weeks=110)
},
{
"item_id": 2,
"created_at": CURRENT_TIMESTAMP - timedelta(weeks=90)
},
{
"item_id": 1,
"created_at": CURRENT_TIMESTAMP - timedelta(weeks=30)
},
{
"item_id": 3,
"created_at": CURRENT_TIMESTAMP - timedelta(weeks=3)
},
{
"item_id": 3,
"created_at": CURRENT_TIMESTAMP
},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_prune_plays(
session,
CURRENT_TIMESTAMP,
cutoff_timestamp=datetime.now() - timedelta(weeks=6),
)
# verify plays
plays_result: List[Play] = session.query(Play).order_by(Play.id).all()
assert len(plays_result) == 2
assert plays_result[0].id == 6
assert plays_result[0].play_item_id == 3
assert plays_result[0].created_at == CURRENT_TIMESTAMP - timedelta(
weeks=3)
assert plays_result[1].id == 7
assert plays_result[1].play_item_id == 3
assert plays_result[1].created_at == CURRENT_TIMESTAMP
# verify archive
plays_archive_result: List[PlaysArchive] = (
session.query(PlaysArchive).order_by(PlaysArchive.id).all())
assert len(plays_archive_result) == 5
assert plays_archive_result[0].id == 1
assert plays_archive_result[0].play_item_id == 1
assert plays_archive_result[0].archived_at == CURRENT_TIMESTAMP
assert plays_archive_result[1].id == 2
assert plays_archive_result[1].play_item_id == 3
assert plays_archive_result[1].archived_at == CURRENT_TIMESTAMP
assert plays_archive_result[2].id == 3
assert plays_archive_result[2].play_item_id == 2
assert plays_archive_result[2].archived_at == CURRENT_TIMESTAMP
assert plays_archive_result[3].id == 4
assert plays_archive_result[3].play_item_id == 2
assert plays_archive_result[3].archived_at == CURRENT_TIMESTAMP
assert plays_archive_result[4].id == 5
assert plays_archive_result[4].play_item_id == 1
assert plays_archive_result[4].archived_at == CURRENT_TIMESTAMP
def test_index_aggregate_plays_same_checkpoint(app):
"""Test that we should not update when last index is the same"""
# setup
with app.app_context():
db = get_db()
# run
entities = {
"tracks": [
{
"track_id": 1,
"title": "track 1"
},
{
"track_id": 2,
"title": "track 2"
},
{
"track_id": 3,
"title": "track 3"
},
{
"track_id": 4,
"title": "track 4"
},
],
"aggregate_plays": [
# Current Plays
{
"play_item_id": 1,
"count": 3
},
{
"play_item_id": 2,
"count": 3
},
{
"play_item_id": 3,
"count": 3
},
],
"indexing_checkpoints": [{
"tablename": "aggregate_plays",
"last_checkpoint": 9
}],
"plays": [
# Current Plays
{
"item_id": 1
},
{
"item_id": 1
},
{
"item_id": 1
},
{
"item_id": 2
},
{
"item_id": 2
},
{
"item_id": 2
},
{
"item_id": 3
},
{
"item_id": 3
},
{
"item_id": 3
},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_update_aggregate_plays(session)
results: List[AggregatePlays] = (
session.query(AggregatePlays).order_by(
AggregatePlays.play_item_id).all())
assert len(results) == 3
def test_index_aggregate_plays_populate(app):
"""Test that we should populate plays from empty"""
date = datetime.now()
# setup
with app.app_context():
db = get_db()
# run
entities = {
"tracks": [
{
"track_id": 0,
"title": "track 0"
},
{
"track_id": 1,
"title": "track 1"
},
{
"track_id": 2,
"title": "track 2"
},
{
"track_id": 3,
"title": "track 3"
},
{
"track_id": 4,
"title": "track 4"
},
],
"plays": [
# Current Plays
{
"item_id": 0
},
{
"item_id": 0
},
{
"item_id": 1
},
{
"item_id": 1
},
{
"item_id": 2
},
{
"item_id": 3
},
# > 1 wk plays
{
"item_id": 2,
"created_at": date - timedelta(weeks=2)
},
{
"item_id": 2,
"created_at": date - timedelta(weeks=2)
},
{
"item_id": 3,
"created_at": date - timedelta(weeks=2)
},
{
"item_id": 3
},
{
"item_id": 3
},
{
"item_id": 4
},
{
"item_id": 4
},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_update_aggregate_plays(session)
results: List[AggregatePlays] = (
session.query(AggregatePlays).order_by(
AggregatePlays.play_item_id).all())
assert len(results) == 5
assert results[0].play_item_id == 0
assert results[0].count == 2
assert results[1].play_item_id == 1
assert results[1].count == 2
assert results[2].play_item_id == 2
assert results[2].count == 3
assert results[3].play_item_id == 3
assert results[3].count == 4
assert results[4].play_item_id == 4
assert results[4].count == 2
def test_calculate_related_artists_scores(app):
with app.app_context():
db = get_db()
populate_mock_db(db, entities)
with db.scoped_session() as session:
_update_aggregate_user(session)
# Check sampled (with large enough sample to get all rows for deterministic result)
rows = _calculate_related_artists_scores(
session,
0,
sample_size=200 + 50 + 100 + 40 + 5 + 500 +
200, # sum of all the follows
)
assert rows[0].related_artist_user_id == 1 and math.isclose(
rows[0].score, 50, abs_tol=0.001)
assert rows[1].related_artist_user_id == 2 and math.isclose(
rows[1].score, 25, abs_tol=0.001)
assert rows[2].related_artist_user_id == 6 and math.isclose(
rows[2].score, 18, abs_tol=0.001)
assert rows[3].related_artist_user_id == 3 and math.isclose(
rows[3].score, 10, abs_tol=0.001)
assert rows[4].related_artist_user_id == 5 and math.isclose(
rows[4].score, 5, abs_tol=0.001)
assert rows[5].related_artist_user_id == 4 and math.isclose(
rows[5].score, 3.2, abs_tol=0.001)
# Check unsampled
rows = _calculate_related_artists_scores(session, 0)
assert rows[0].related_artist_user_id == 1 and math.isclose(
rows[0].score, 50, abs_tol=0.001)
assert rows[1].related_artist_user_id == 2 and math.isclose(
rows[1].score, 25, abs_tol=0.001)
assert rows[2].related_artist_user_id == 6 and math.isclose(
rows[2].score, 18, abs_tol=0.001)
assert rows[3].related_artist_user_id == 3 and math.isclose(
rows[3].score, 10, abs_tol=0.001)
assert rows[4].related_artist_user_id == 5 and math.isclose(
rows[4].score, 5, abs_tol=0.001)
assert rows[5].related_artist_user_id == 4 and math.isclose(
rows[5].score, 3.2, abs_tol=0.001)
# Check edge case with 0 followers
populate_mock_db(
db,
{"follows": [{
"follower_user_id": 100,
"followee_user_id": 7
}]})
rows = _calculate_related_artists_scores(session, 0)
# Same results as unsampled. Shouldn't throw DivideByZero exception
assert rows[0].related_artist_user_id == 1 and math.isclose(
rows[0].score, 50, abs_tol=0.001)
assert rows[1].related_artist_user_id == 2 and math.isclose(
rows[1].score, 25, abs_tol=0.001)
assert rows[2].related_artist_user_id == 6 and math.isclose(
rows[2].score, 18, abs_tol=0.001)
assert rows[3].related_artist_user_id == 3 and math.isclose(
rows[3].score, 10, abs_tol=0.001)
assert rows[4].related_artist_user_id == 5 and math.isclose(
rows[4].score, 5, abs_tol=0.001)
assert rows[5].related_artist_user_id == 4 and math.isclose(
rows[5].score, 3.2, abs_tol=0.001)
def test_get_repost_feed_for_user(app):
"""Tests that a repost feed for a user can be queried"""
with app.app_context():
db = get_db()
test_entities = {
"reposts": [
# Note these reposts are in chronological order in addition
# so the repost feed should pull them "backwards" for reverse chronological
# sort order.
{
"user_id": 1,
"repost_item_id": 5,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 2,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 3,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 1,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 4,
"repost_type": "track"
},
{
"user_id": 1,
"repost_item_id": 4,
"repost_type": "playlist"
},
{
"user_id": 1,
"repost_item_id": 8,
"repost_type": "album"
},
{
"user_id": 1,
"repost_item_id": 6,
"repost_type": "track"
},
],
"tracks": [
{
"track_id": 1,
"title": "track 1"
},
{
"track_id": 2,
"title": "track 2"
},
{
"track_id": 3,
"title": "track 3"
},
{
"track_id": 4,
"title": "track 4"
},
{
"track_id": 5,
"title": "track 5"
},
{
"track_id": 6,
"title": "track 6"
},
{
"track_id": 7,
"title": "track 7"
},
{
"track_id": 8,
"title": "track 8"
},
],
"playlists": [
{
"playlist_id": 1,
"playlist_name": "playlist 1"
},
{
"playlist_id": 2,
"playlist_name": "playlist 2"
},
{
"playlist_id": 3,
"playlist_name": "playlist 3"
},
{
"playlist_id": 4,
"playlist_name": "playlist 4"
},
{
"playlist_id": 5,
"playlist_name": "playlist 5"
},
{
"playlist_id": 6,
"playlist_name": "playlist 6"
},
{
"playlist_id": 7,
"playlist_name": "playlist 7"
},
{
"playlist_id": 8,
"playlist_name": "album 8"
},
],
}
populate_mock_db(db, test_entities)
with db.scoped_session() as session:
repost_feed = _get_repost_feed_for_user(session, 1, {
"limit": 10,
"offset": 0
})
assert repost_feed[0]["title"] == "track 6"
assert repost_feed[1]["playlist_name"] == "album 8"
assert repost_feed[2]["playlist_name"] == "playlist 4"
assert repost_feed[3]["title"] == "track 4"
assert repost_feed[4]["title"] == "track 1"
assert repost_feed[5]["title"] == "track 3"
assert repost_feed[6]["title"] == "track 2"
assert repost_feed[7]["title"] == "track 5"
def test_fetch_and_parse_sol_rewards_transfer_instruction(app): # pylint: disable=W0621
with app.app_context():
db = get_db()
redis = get_redis()
solana_client_manager_mock = create_autospec(SolanaClientManager)
solana_client_manager_mock.get_sol_tx_info.return_value = mock_tx_info
first_tx_sig = "tx_sig_one"
second_tx_sig = "tx_sig_two"
parsed_tx = fetch_and_parse_sol_rewards_transfer_instruction(
solana_client_manager_mock, first_tx_sig)
assert (parsed_tx["transfer_instruction"]["amount"] # pylint: disable=E1136
== 10000000000)
assert (parsed_tx["transfer_instruction"]["eth_recipient"] # pylint: disable=E1136
== "0x0403be3560116a12b467855cb29a393174a59876")
assert (parsed_tx["transfer_instruction"]["challenge_id"] # pylint: disable=E1136
== "profile-completion")
assert parsed_tx["tx_sig"] == first_tx_sig
assert parsed_tx["slot"] == 72131741
test_user_entries = {
"users": [
{
"user_id": 1,
"handle": "piazza",
"wallet": "0x0403be3560116a12b467855cb29a393174a59876",
},
]
}
with db.scoped_session() as session:
process_batch_sol_reward_manager_txs(session, [parsed_tx], redis)
disbursments = session.query(ChallengeDisbursement).all()
assert len(disbursments) == 1
disbursement = disbursments[0]
# Assert that this invalid user was set to user_id 0
assert disbursement.user_id == 0
reward_manager_tx_1 = (session.query(RewardManagerTransaction).filter(
RewardManagerTransaction.signature == first_tx_sig).all())
assert len(reward_manager_tx_1) == 1
populate_mock_db(db, test_user_entries)
parsed_tx["tx_sig"] = second_tx_sig
next_slot = parsed_tx["slot"] + 1
parsed_tx["slot"] = next_slot
parsed_tx["transfer_instruction"]["challenge_id"] = "tt"
with db.scoped_session() as session:
process_batch_sol_reward_manager_txs(session, [parsed_tx], redis)
disbursments = (session.query(ChallengeDisbursement).order_by(
desc(ChallengeDisbursement.slot)).all())
reward_manager_tx_1 = (session.query(RewardManagerTransaction).filter(
RewardManagerTransaction.signature == second_tx_sig).all())
assert len(reward_manager_tx_1) == 1
assert len(disbursments) == 2
disbursment = disbursments[0]
assert disbursment.challenge_id == "tt"
assert disbursment.user_id == 1
assert disbursment.signature == second_tx_sig
assert disbursment.slot == next_slot
assert disbursment.specifier == "123456789"
reward_manager_tx_2 = (session.query(RewardManagerTransaction).filter(
RewardManagerTransaction.signature == second_tx_sig).all())
assert len(reward_manager_tx_2) == 1
