def _index_aggregate_monthly_plays(session):
# get the last updated id that counted towards the current aggregate monthly plays
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_MONTHLY_PLAYS_TABLE_NAME)
# get the new latest checkpoint
new_id_checkpoint = (session.query(func.max(Play.id))).scalar()
if not new_id_checkpoint or new_id_checkpoint == prev_id_checkpoint:
logger.info(
"index_aggregate_monthly_plays.py | Skip update because there are no new plays"
)
return
# update aggregate monthly plays with new plays that came after the prev_id_checkpoint
# group into monthly buckets
# insert / update those buckets into table
logger.info(
f"index_aggregate_monthly_plays.py | Updating {AGGREGATE_MONTHLY_PLAYS_TABLE_NAME}"
)
session.execute(
text(UPSERT_AGGREGATE_MONTHLY_PLAYS_QUERY),
{
"prev_id_checkpoint": prev_id_checkpoint,
"new_id_checkpoint": new_id_checkpoint,
},
)
# update indexing_checkpoints with the new id
save_indexed_checkpoint(session, AGGREGATE_MONTHLY_PLAYS_TABLE_NAME,
new_id_checkpoint)
def basic_tests(session, last_checkpoint=12, previous_count=0):
"""Helper for testing the basic_entities as is"""
# read from aggregate_track table
results: List[AggregateTrack] = (session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert len(results) == 4
assert results[0].track_id == 1
assert results[0].repost_count == previous_count + 3
assert results[0].save_count == previous_count + 1
assert results[1].track_id == 2
assert results[1].repost_count == previous_count + 0
assert results[1].save_count == previous_count + 0
assert results[2].track_id == 4
assert results[2].repost_count == previous_count + 0
assert results[2].save_count == previous_count + 4
assert results[3].track_id == 5
assert results[3].repost_count == previous_count + 0
assert results[3].save_count == previous_count + 0
prev_id_checkpoint = get_last_indexed_checkpoint(session, AGGREGATE_TRACK)
assert prev_id_checkpoint == last_checkpoint
def test_index_aggregate_user_same_checkpoint(app):
"""Test that we should not update when last index is the same"""
with app.app_context():
db = get_db()
entities = deepcopy(basic_entities)
current_blocknumber = basic_entities["blocks"][0]["number"]
entities.update({
"indexing_checkpoints": [{
"tablename": AGGREGATE_USER,
"last_checkpoint": current_blocknumber
}],
})
populate_mock_db(db, entities)
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert len(results) == 0
_update_aggregate_user(session)
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert len(results) == 0
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 3
def basic_tests(session, last_checkpoint=6):
"""Helper for testing the basic_entities as is"""
# read from aggregate_user table
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert len(results) == 2
assert results[0].user_id == 1
assert results[0].track_count == 2
assert results[0].playlist_count == 1
assert results[0].album_count == 1
assert results[0].follower_count == 1
assert results[0].following_count == 1
assert results[0].repost_count == 0
assert results[0].track_save_count == 0
assert results[1].user_id == 2
assert results[1].track_count == 1
assert results[1].playlist_count == 0
assert results[1].album_count == 0
assert results[1].follower_count == 1
assert results[1].following_count == 1
assert results[1].repost_count == 2
assert results[1].track_save_count == 1
prev_id_checkpoint = get_last_indexed_checkpoint(session, AGGREGATE_USER)
assert prev_id_checkpoint == last_checkpoint
def test_index_aggregate_track_empty_tracks(app):
"""Test that track metadata without tracks table won't break"""
with app.app_context():
db = get_db()
entities = {
"users": [],
"indexing_checkpoints": [{
"tablename": AGGREGATE_TRACK,
"last_checkpoint": 0
}],
"tracks": [],
"reposts": [
{
"repost_item_id": 1,
"repost_type": "track",
"track_id": 1
},
{
"repost_item_id": 1,
"repost_type": "playlist",
"track_id": 1
},
],
"saves": [
{
"save_item_id": 1,
"save_type": "track",
"track_id": 1
},
{
"save_item_id": 1,
"save_type": "playlist",
"track_id": 1
},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_update_aggregate_track(session)
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert (len(results) == 0
), "Test that without Tracks there will be no AggregateTracks"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_TRACK)
assert prev_id_checkpoint == 1
def update_aggregate_table(
logger,
session,
table_name,
query,
checkpoint_name,
current_checkpoint,
):
metric = PrometheusMetric(
"update_aggregate_table_latency_seconds",
"Runtimes for src.task.aggregates:update_aggregate_table()",
("table_name", "task_name"),
)
# get name of the caller function
task_name = f"{currentframe().f_back.f_code.co_name}()"
# get the last updated id that counted towards the current aggregate track
prev_checkpoint = get_last_indexed_checkpoint(session, table_name)
if not current_checkpoint or current_checkpoint == prev_checkpoint:
logger.info(
f"{task_name} | Skipping aggregation update because there are no new blocks"
f" | checkpoint: ({prev_checkpoint}, {current_checkpoint}]")
return
# update aggregate track with new tracks that came after the prev_checkpoint
logger.info(f"{task_name} | Updating {table_name}"
f" | checkpoint: ({prev_checkpoint}, {current_checkpoint}]")
session.execute(
text(query),
{
f"prev_{checkpoint_name}": prev_checkpoint,
f"current_{checkpoint_name}": current_checkpoint,
},
)
metric.save_time({"table_name": table_name, "task_name": task_name})
# update indexing_checkpoints with the new id
save_indexed_checkpoint(session, table_name, current_checkpoint)
def test_index_aggregate_user_empty_completely(app):
"""Test a completely empty database won't break"""
with app.app_context():
db = get_db()
entities = {}
populate_mock_db(db, entities, block_offset=3)
with db.scoped_session() as session:
_update_aggregate_user(session)
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (len(results) == 0
), "Test that empty entities won't generate AggregateUsers"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 0
def _index_hourly_play_counts(session):
# get checkpoints
prev_id_checkpoint = get_last_indexed_checkpoint(
session, HOURLY_PLAY_COUNTS_TABLE_NAME)
new_id_checkpoint = (session.query(func.max(Play.id))).scalar()
if not new_id_checkpoint or new_id_checkpoint == prev_id_checkpoint:
logger.info(
"index_hourly_play_counts.py | Skip update because there are no new plays"
)
return
# get play counts in hourly buckets
hourly_play_counts: List[HourlyPlayCounts] = (session.query(
func.date_trunc("hour", Play.created_at).label("hourly_timestamp"),
func.count(Play.id).label("play_count"),
).filter(Play.id > prev_id_checkpoint).filter(
Play.id <= new_id_checkpoint).group_by(
func.date_trunc("hour", Play.created_at)).order_by(
desc("hourly_timestamp")).all())
# upsert hourly play count
# on first population, this will execute an insert for each hour
# subsequent updates should include 1 or 2 upserts
for hourly_play_count in hourly_play_counts:
session.execute(
UPSERT_HOURLY_PLAY_COUNTS_QUERY,
{
"hourly_timestamp": hourly_play_count.hourly_timestamp,
"play_count": hourly_play_count.play_count,
},
)
# update with new checkpoint
save_indexed_checkpoint(session, HOURLY_PLAY_COUNTS_TABLE_NAME,
new_id_checkpoint)
def _update_aggregate_tips(session: Session, redis: Redis):
latest_user_bank_slot = redis.get(latest_sol_user_bank_slot_key)
prev_slot = get_last_indexed_checkpoint(session, AGGREGATE_TIPS)
max_slot_result = session.query(func.max(UserTip.slot)).one()
max_slot = int(max_slot_result[0]) if max_slot_result[0] is not None else 0
if prev_slot == max_slot:
if latest_user_bank_slot is not None:
redis.set(latest_sol_aggregate_tips_slot_key, int(latest_user_bank_slot))
return
ranks_before = _get_ranks(session, prev_slot, max_slot)
update_aggregate_table(
logger,
session,
AGGREGATE_TIPS,
UPDATE_AGGREGATE_USER_TIPS_QUERY,
"slot",
max_slot,
)
ranks_after = _get_ranks(session, prev_slot, max_slot)
index_rank_ups(session, ranks_before, ranks_after, max_slot)
if latest_user_bank_slot is not None:
redis.set(latest_sol_aggregate_tips_slot_key, int(latest_user_bank_slot))
def _index_user_listening_history(session):
# get the last updated id that counted towards user_listening_history
# use as lower bound
prev_id_checkpoint = get_last_indexed_checkpoint(
session, USER_LISTENING_HISTORY_TABLE_NAME)
# get new plays since the last checkpoint
new_plays = (session.query(
Play.id, Play.user_id, Play.play_item_id,
Play.created_at).filter(Play.id > prev_id_checkpoint).filter(
Play.user_id != None).order_by(sa.asc(
Play.id)).limit(BATCH_SIZE)).all()
# no update exit early
if not new_plays:
return
new_checkpoint = new_plays[-1].id # get the highest play id
# get existing user listening history for users with new plays
users_with_new_plays = {new_play.user_id for new_play in new_plays}
existing_user_listening_history = (
session.query(UserListeningHistory).filter(
UserListeningHistory.user_id.in_(users_with_new_plays)).all())
existing_users = {
user_history.user_id
for user_history in existing_user_listening_history
}
# reduce new plays
insert_user_listening_history_dict = DefaultDict(list)
update_user_listening_history_dict = DefaultDict(list)
for new_play in reversed(new_plays):
listen_history = ListenHistory(new_play.play_item_id,
new_play.created_at).to_dict()
if new_play.user_id in existing_users:
update_user_listening_history_dict[new_play.user_id].append(
listen_history)
else:
insert_user_listening_history_dict[new_play.user_id].append(
listen_history)
# make updates to existing users
for i, user_history in enumerate(existing_user_listening_history):
deduped_history_dict = {}
for existing_play in reversed(user_history.listening_history):
deduped_history_dict[
existing_play["track_id"]] = existing_play["timestamp"]
for new_play in reversed(
update_user_listening_history_dict[user_history.user_id]):
deduped_history_dict[new_play["track_id"]] = new_play["timestamp"]
existing_user_listening_history[
i].listening_history = sort_listening_history(deduped_history_dict)
# insert for new users
new_user_listening_history = []
for user, listening_history in insert_user_listening_history_dict.items():
deduped_history_dict = {}
for new_play in reversed(listening_history):
deduped_history_dict[new_play["track_id"]] = new_play["timestamp"]
new_user_listening_history.append(
UserListeningHistory(
user_id=user,
listening_history=sort_listening_history(deduped_history_dict),
))
session.add_all(new_user_listening_history)
# update indexing_checkpoints with the new id
save_indexed_checkpoint(session, USER_LISTENING_HISTORY_TABLE_NAME,
new_checkpoint)
def test_index_aggregate_user_update_with_only_aggregate_user(app):
"""Test that aggregate_user will never be truncated even when no other data"""
with app.app_context():
db = get_db()
entities = {
"aggregate_user": [
{
"user_id": 1,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
{
"user_id": 2,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
{
"user_id": 3,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
],
"indexing_checkpoints": [{
"tablename": AGGREGATE_USER,
"last_checkpoint": 9
}],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert len(results) == 3, "Test that entities exist as expected"
_update_aggregate_user(session)
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (
len(results) == 3
), "Test zero-modifications since last_checkpoint is in the future"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 9
entities = {
"indexing_checkpoints": [{
"tablename": AGGREGATE_USER,
"last_checkpoint": 0
}],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (
len(results) == 3
), "Test that entities exist as expected, even though checkpoint has been reset"
_update_aggregate_user(session)
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (
len(results) == 3
), "Test that aggregate_user has not been truncated due to reset checkpoint"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 0
def test_index_aggregate_user_update_with_extra_user(app):
"""Test that the entire aggregate_user table is not truncated"""
with app.app_context():
db = get_db()
entities = deepcopy(basic_entities)
entities.update({
"indexing_checkpoints": [{
"tablename": AGGREGATE_USER,
"last_checkpoint": 0
}],
"aggregate_user": [
{
"user_id": 1,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
{
"user_id": 2,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
{
"user_id": 3,
"track_count": 9,
"playlist_count": 9,
"album_count": 9,
"follower_count": 9,
"following_count": 9,
"repost_count": 9,
"track_save_count": 9,
},
],
})
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert len(results) == 0, "Test that we start with clean tables"
populate_mock_db(db, entities)
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert len(
results) == 3, "Test that aggregate_user entities are populated"
_update_aggregate_user(session)
with db.scoped_session() as session:
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert len(results) == 3
assert results[0].user_id == 1
assert results[0].track_count == 2
assert results[0].playlist_count == 1
assert results[0].album_count == 1
assert results[0].follower_count == 1
assert results[0].following_count == 1
assert results[0].repost_count == 0
assert results[0].track_save_count == 0
assert results[1].user_id == 2
assert results[1].track_count == 1
assert results[1].playlist_count == 0
assert results[1].album_count == 0
assert results[1].follower_count == 1
assert results[1].following_count == 1
assert results[1].repost_count == 2
assert results[1].track_save_count == 1
assert results[2].user_id == 3
assert results[2].track_count == 9
assert results[2].playlist_count == 9
assert results[2].album_count == 9
assert results[2].follower_count == 9
assert results[2].following_count == 9
assert results[2].repost_count == 9
assert results[2].track_save_count == 9
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 3
def test_index_aggregate_user_empty_activity(app):
"""Test that a populated users table without activity won't break"""
with app.app_context():
db = get_db()
entities = {
"users": [
{
"user_id": 1,
"handle": "user1"
},
{
"user_id": 2,
"handle": "user2"
},
],
"indexing_checkpoints": [{
"tablename": AGGREGATE_USER,
"last_checkpoint": 5
}],
}
# create user1 and user2 in blocknumbers 3 and 4, respectively
populate_mock_db(db, entities, block_offset=3)
with db.scoped_session() as session:
_update_aggregate_user(session)
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (
len(results) == 0
), "Test that users updated on blocks previous to '5' will not be targeted"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 4
entities = {
"indexing_checkpoints": [{
"tablename": AGGREGATE_USER,
"last_checkpoint": 1
}],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_update_aggregate_user(session)
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (
len(results) == 2
), "Test that users updated on blocks after '1' will be targeted"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 4
def test_index_aggregate_user_empty_users(app):
"""Test that user metadata without users table won't break"""
with app.app_context():
db = get_db()
entities = {
"users": [],
"indexing_checkpoints": [{
"tablename": AGGREGATE_USER,
"last_checkpoint": 0
}],
"tracks": [
{
"track_id": 1,
"owner_id": 1
},
{
"track_id": 2,
"owner_id": 1
},
{
"track_id": 3,
"is_unlisted": True,
"owner_id": 1
},
],
"playlists": [
{
"playlist_id": 1,
"playlist_owner_id": 1,
"playlist_name": "name",
"description": "description",
"playlist_contents": {
"track_ids": [
{
"track": 1,
"time": 1
},
{
"track": 2,
"time": 2
},
{
"track": 3,
"time": 3
},
]
},
},
{
"playlist_id": 2,
"is_album": True,
"playlist_owner_id": 1,
"playlist_name": "name",
"description": "description",
"playlist_contents": {
"track_ids": [
{
"track": 1,
"time": 1
},
{
"track": 2,
"time": 2
},
{
"track": 3,
"time": 3
},
]
},
},
],
"follows": [
{
"follower_user_id": 1,
"followee_user_id": 2,
"created_at": datetime.now() - timedelta(days=8),
},
{
"follower_user_id": 2,
"followee_user_id": 1,
"created_at": datetime.now() - timedelta(days=8),
},
],
"reposts": [
{
"repost_item_id": 1,
"repost_type": "track",
"user_id": 1
},
{
"repost_item_id": 1,
"repost_type": "playlist",
"user_id": 1
},
],
"saves": [
{
"save_item_id": 1,
"save_type": "track",
"user_id": 1
},
{
"save_item_id": 1,
"save_type": "playlist",
"user_id": 1
},
],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_update_aggregate_user(session)
results: List[AggregateUser] = (session.query(AggregateUser).order_by(
AggregateUser.user_id).all())
assert (len(results) == 0
), "Test that without Users there will be no AggregateUsers"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_USER)
assert prev_id_checkpoint == 2
def test_index_aggregate_track_populate(app):
"""Test that we should populate tracks from empty"""
with app.app_context():
db = get_db()
with db.scoped_session() as session:
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert (len(results) == 0
), "Test aggregate_track is empty before populate_mock_db()"
# create db entries based on entities
populate_mock_db(db, basic_entities, block_offset=3)
last_checkpoint = 12
with db.scoped_session() as session:
# confirm nothing exists before _update_aggregate_track()
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert (
len(results) == 0
), "Test aggregate_track is empty before _update_aggregate_track()"
# trigger celery task
_update_aggregate_track(session)
# run basic tests against basic_entities
basic_tests(session, last_checkpoint=last_checkpoint)
# delete a track
entities = {
"tracks": [
{
"track_id": 2,
"owner_id": 1,
"is_current": True,
"is_delete": True,
},
],
}
populate_mock_db(db, entities)
last_checkpoint += 1
# confirm track 2 no longer has a row in aggregate_track
with db.scoped_session() as session:
_update_aggregate_track(session)
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert len(results) == 3
assert results[0].track_id == 1
assert results[0].repost_count == 3
assert results[0].save_count == 1
assert results[1].track_id == 4
assert results[1].repost_count == 0
assert results[1].save_count == 4
assert results[2].track_id == 5
assert results[2].repost_count == 0
assert results[2].save_count == 0
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_TRACK)
assert prev_id_checkpoint == last_checkpoint
# repost a deleted track
entities = {
"reposts": [
{
"repost_item_id": 2,
"repost_type": "track",
"user_id": 2,
"is_current": True,
},
],
}
populate_mock_db(db, entities)
last_checkpoint += 1
# confirm track 2 still no longer has a row in aggregate_track
with db.scoped_session() as session:
_update_aggregate_track(session)
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert len(results) == 3
assert results[0].track_id == 1
assert results[0].repost_count == 3
assert results[0].save_count == 1
assert results[1].track_id == 4
assert results[1].repost_count == 0
assert results[1].save_count == 4
assert results[2].track_id == 5
assert results[2].repost_count == 0
assert results[2].save_count == 0
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_TRACK)
assert prev_id_checkpoint == last_checkpoint
# undelete a track
entities = {
"tracks": [
{
"track_id": 2,
"owner_id": 1,
"is_current": True,
"is_delete": False,
},
],
}
populate_mock_db(db, entities)
last_checkpoint += 1
# confirm track 2 has a row in aggregate_track again, with an additional repost
with db.scoped_session() as session:
_update_aggregate_track(session)
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert len(results) == 4
assert results[0].track_id == 1
assert results[0].repost_count == 3
assert results[0].save_count == 1
assert results[1].track_id == 2
assert results[1].repost_count == 1
assert results[1].save_count == 0
assert results[2].track_id == 4
assert results[2].repost_count == 0
assert results[2].save_count == 4
assert results[3].track_id == 5
assert results[3].repost_count == 0
assert results[3].save_count == 0
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_TRACK)
assert prev_id_checkpoint == last_checkpoint
def test_index_aggregate_track_update_with_only_aggregate_track(app):
"""Test that aggregate_track will not be manipulated when there is no other data"""
with app.app_context():
db = get_db()
entities = {
"aggregate_track": [
{
"track_id": 1,
"repost_count": 9,
"save_count": 9,
},
{
"track_id": 2,
"repost_count": 9,
"save_count": 9,
},
{
"track_id": 3,
"repost_count": 9,
"save_count": 9,
},
],
"indexing_checkpoints": [{
"tablename": AGGREGATE_TRACK,
"last_checkpoint": 9
}],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert len(results) == 3, "Test that entities exist as expected"
_update_aggregate_track(session)
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert (
len(results) == 3
), "Test zero-modifications since last_checkpoint is in the future"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_TRACK)
assert prev_id_checkpoint == 9
entities = {
"indexing_checkpoints": [{
"tablename": AGGREGATE_TRACK,
"last_checkpoint": 0
}],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert (
len(results) == 3
), "Test that entities exist as expected, even though checkpoint has been reset"
_update_aggregate_track(session)
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert (
len(results) == 3
), "Test that aggregate_track has not been changed due to lack of track data"
for result in results:
assert (
result.repost_count == 9
), "Test that aggregate_track has not been changed due to lack of track data"
assert (
result.save_count == 9
), "Test that aggregate_track has not been changed due to lack of track data"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_TRACK)
assert prev_id_checkpoint == 0
def test_index_aggregate_track_empty_activity(app):
"""Test that a populated tracks table without activity won't break"""
with app.app_context():
db = get_db()
entities = {
"tracks": [
{
"track_id": 1,
"owner_id": 1,
"is_current": True
},
{
"track_id": 2,
"owner_id": 1,
"is_current": True
},
{
"track_id": 3,
"owner_id": 1,
"is_current": True,
"is_delete": True,
},
{
"track_id": 4,
"owner_id": 2,
"is_current": True
},
{
"track_id": 5,
"owner_id": 1,
"is_current": True,
"is_unlisted": True,
},
],
"indexing_checkpoints": [{
"tablename": AGGREGATE_TRACK,
"last_checkpoint": 10
}],
}
populate_mock_db(db, entities, block_offset=6)
with db.scoped_session() as session:
_update_aggregate_track(session)
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert (
len(results) == 0
), "Test that tracks updated on blocks previous to '10' will not be targeted"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_TRACK)
assert prev_id_checkpoint == 10
entities = {
"indexing_checkpoints": [{
"tablename": AGGREGATE_TRACK,
"last_checkpoint": 1
}],
}
populate_mock_db(db, entities)
with db.scoped_session() as session:
_update_aggregate_track(session)
results: List[AggregateTrack] = (
session.query(AggregateTrack).order_by(
AggregateTrack.track_id).all())
assert (
len(results) == 4
), "Test that tracks updated on blocks after '1' will be targeted"
prev_id_checkpoint = get_last_indexed_checkpoint(
session, AGGREGATE_TRACK)
assert prev_id_checkpoint == 10