def lookup_track_record(update_task, session, entry, event_track_id,
block_number, block_hash):
# Check if track record exists
track_exists = (
session.query(Track).filter_by(track_id=event_track_id).count() > 0)
track_record = None
if track_exists:
track_record = (session.query(Track).filter(
Track.track_id == event_track_id,
Track.is_current == True).first())
# expunge the result from sqlalchemy so we can modify it without UPDATE statements being made
# https://stackoverflow.com/questions/28871406/how-to-clone-a-sqlalchemy-db-object-with-new-primary-key
session.expunge(track_record)
make_transient(track_record)
else:
track_record = Track(track_id=event_track_id,
is_current=True,
is_delete=False)
# update block related fields regardless of type
track_record.blocknumber = block_number
track_record.blockhash = block_hash
return track_record
def populate_mock_db(db, test_tracks, date):
"""Helper function to populate thee mock DB with plays"""
with db.scoped_session() as session:
for i, track_meta in enumerate(test_tracks):
blockhash = hex(i)
block = Block(
blockhash=blockhash,
number=i,
parenthash="0x01",
is_current=(i == 0),
)
track = Track(
blockhash=hex(i),
blocknumber=i,
track_id=i,
is_current=track_meta.get("is_current", True),
is_delete=track_meta.get("is_delete", False),
owner_id=300,
route_id="",
track_segments=[],
genre=track_meta.get("genre", ""),
updated_at=track_meta.get("updated_at", date),
created_at=track_meta.get("created_at", date),
is_unlisted=track_meta.get("is_unlisted", False),
)
# add block and then flush before
# adding track, bc track.blocknumber foreign key
# references block
session.add(block)
session.flush()
session.add(track)
def test_link_bonus_track(self):
a = Artist(name="Atmosphere")
result = Track(name="Idiot - Bonus Track")
result.primary_artists.extend([a])
_artists = result.primary_artists if len(result.primary_artists) > 0 else result.featured_artists
url = GenUtils.link([_artist.name for _artist in _artists], result.name)
self.assertTrue(url.endswith("/Atmosphere-idiot-lyrics"))
def test_link_slabed_2(self):
a = Artist(name="Z-Ro")
result = Track(name="Slabed - Still Watchin")
result.featured_artists.extend([a])
_artists = result.primary_artists if len(result.primary_artists) > 0 else result.featured_artists
url = GenUtils.link([_artist.name for _artist in _artists], result.name)
self.assertTrue(url.endswith("/Z-ro-still-watchin-lyrics"))
def test_link_slabed(self):
a = Artist(name="Z-Ro")
result = Track(name="Swagger Like Us (G Mix) - S.L.A.B - Ed")
result.featured_artists.extend([a])
_artists = result.primary_artists if len(result.primary_artists) > 0 else result.featured_artists
url = GenUtils.link([_artist.name for _artist in _artists], result.name)
self.assertTrue(url.endswith("/Z-ro-swagger-like-us-g-mix-lyrics"))
def test_link_chopped_and_feat(self):
a = Artist(name="Z-Ro")
result = Track(name="Sunshine - Chopped (feat. Lil’ Keke)")
result.featured_artists.extend([a])
_artists = result.primary_artists if len(result.primary_artists) > 0 else result.featured_artists
url = GenUtils.link([_artist.name for _artist in _artists], result.name)
self.assertTrue(url.endswith("/Z-ro-sunshine-lyrics"))
def setup_trending(db):
# pylint: disable=W0621
with db.scoped_session() as session:
# seed tracks + blocks
for i, track_meta in enumerate(test_tracks):
blockhash = hex(i)
block = Block(
blockhash=blockhash,
number=i,
parenthash='0x01',
is_current=True,
)
track = Track(blockhash=blockhash,
blocknumber=i,
track_id=i,
is_current=track_meta.get("is_current", True),
is_delete=track_meta.get("is_delete", False),
owner_id=300,
route_id='',
track_segments=[],
genre=track_meta.get("genre", ""),
updated_at=track_meta.get("updated_at",
datetime.now()),
created_at=track_meta.get("created_at",
datetime.now()),
is_unlisted=track_meta.get("is_unlisted", False))
# add block and then flush before
# adding track, bc track.blocknumber foreign key
# references block
session.add(block)
session.flush()
session.add(track)
# seed plays
for i, play_meta in enumerate(test_plays):
play = Play(id=i,
play_item_id=play_meta.get("item_id"),
created_at=play_meta.get("created_at", datetime.now()))
session.add(play)
def setup_trending(db, date):
# Test data
# test tracks
# when creating tracks, track_id == index
test_tracks = [
{"genre": "Electronic"},
{"genre": "Pop"},
{"genre": "Electronic"},
# Tracks we don't want to count
{"genre": "Electronic", "is_unlisted": True},
{"genre": "Electronic", "is_delete": True},
]
test_plays = [
# Current Plays
{"item_id": 0},
{"item_id": 0},
{"item_id": 1},
{"item_id": 1},
{"item_id": 2},
{"item_id": 3},
# > 1 wk plays
{"item_id": 2, "created_at": date - timedelta(weeks=2)},
{"item_id": 2, "created_at": date - timedelta(weeks=2)},
{"item_id": 3, "created_at": date - timedelta(weeks=2)},
# We don't want to count these guys (tracks deleted/unlisted)
{"item_id": 3},
{"item_id": 3},
{"item_id": 4},
{"item_id": 4},
]
# pylint: disable=W0621
with db.scoped_session() as session:
# seed tracks + blocks
for i, track_meta in enumerate(test_tracks):
blockhash = hex(i)
block = Block(
blockhash=blockhash,
number=i,
parenthash="0x01",
is_current=True,
)
track = Track(
blockhash=blockhash,
blocknumber=i,
track_id=i,
is_current=track_meta.get("is_current", True),
is_delete=track_meta.get("is_delete", False),
owner_id=300,
route_id="",
track_segments=[],
genre=track_meta.get("genre", ""),
updated_at=track_meta.get("updated_at", date),
created_at=track_meta.get("created_at", date),
is_unlisted=track_meta.get("is_unlisted", False),
)
# add block and then flush before
# adding track, bc track.blocknumber foreign key
# references block
session.add(block)
session.flush()
session.add(track)
# seed plays
aggregate_plays = {}
for i, play_meta in enumerate(test_plays):
item_id = play_meta.get("item_id")
if item_id in aggregate_plays:
aggregate_plays[item_id] += 1
else:
aggregate_plays[item_id] = 1
play = Play(
id=i, play_item_id=item_id, created_at=play_meta.get("created_at", date)
)
session.add(play)
for i, count in aggregate_plays.items():
session.add(AggregatePlays(play_item_id=i, count=count))
def setup_search(app_module):
with app_module.app_context():
db = get_db()
# Import app so that it'll run migrations against the db
now = datetime.now()
blocks = [
Block(
blockhash=hex(1),
number=1,
parenthash="0x01",
is_current=False,
),
Block(
blockhash=hex(2),
number=2,
parenthash="0x02",
is_current=False,
),
Block(
blockhash=hex(3),
number=3,
parenthash="0x03",
is_current=True,
),
]
tracks = [
Track(
blockhash=hex(1),
blocknumber=1,
track_id=1,
is_current=True,
is_delete=False,
owner_id=1,
route_id="",
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="the track 1",
download={"cid": None, "is_downloadable": False, "requires_follow": False},
),
Track(
blockhash=hex(2),
blocknumber=2,
track_id=2,
is_current=True,
is_delete=False,
owner_id=2,
route_id="",
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="the track 2",
download={"cid": None, "is_downloadable": True, "requires_follow": False},
),
Track(
blockhash=hex(3),
blocknumber=3,
track_id=3,
is_current=True,
is_delete=False,
owner_id=1,
route_id="",
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="xyz",
download={"cid": None, "is_downloadable": True, "requires_follow": False},
),
]
# need users for the lexeme dict to work
users = [
User(
blockhash=hex(1),
blocknumber=1,
user_id=1,
is_current=True,
handle="",
wallet="",
name="user 1",
updated_at=now,
created_at=now,
),
User(
blockhash=hex(2),
blocknumber=2,
user_id=2,
is_current=True,
handle="",
name="user 2",
wallet="",
updated_at=now,
created_at=now,
),
User(
blockhash=hex(3),
blocknumber=3,
user_id=3,
is_current=True,
handle="",
wallet="",
name="fdwea",
updated_at=now,
created_at=now,
),
]
follows = [
Follow(
blockhash=hex(1),
blocknumber=1,
follower_user_id=2,
followee_user_id=1,
is_current=True,
is_delete=False,
created_at=now,
)
]
playlists = [
Playlist(
blockhash=hex(1),
blocknumber=1,
playlist_id=1,
playlist_owner_id=1,
is_album=False,
is_private=False,
playlist_name="playlist 1",
playlist_contents={"track_ids": [{"track": 1, "time": 1}]},
is_current=True,
is_delete=False,
updated_at=now,
created_at=now,
),
Playlist(
blockhash=hex(2),
blocknumber=2,
playlist_id=2,
playlist_owner_id=2,
is_album=True,
is_private=False,
playlist_name="album 1",
playlist_contents={"track_ids": [{"track": 2, "time": 2}]},
is_current=True,
is_delete=False,
updated_at=now,
created_at=now,
),
]
saves = [
Save(
blockhash=hex(1),
blocknumber=1,
user_id=1,
save_item_id=1,
save_type=SaveType.track,
created_at=now,
is_current=True,
is_delete=False,
),
Save(
blockhash=hex(1),
blocknumber=1,
user_id=1,
save_item_id=1,
save_type=SaveType.playlist,
created_at=now,
is_current=True,
is_delete=False,
),
Save(
blockhash=hex(1),
blocknumber=1,
user_id=1,
save_item_id=2,
save_type=SaveType.album,
created_at=now,
is_current=True,
is_delete=False,
),
]
balances = [
UserBalance(
user_id=1,
balance=0,
associated_wallets_balance=0,
associated_sol_wallets_balance=0,
waudio=0,
)
]
with db.scoped_session() as session:
for block in blocks:
session.add(block)
session.flush()
for track in tracks:
session.add(track)
for user in users:
session.add(user)
session.flush()
for follow in follows:
session.add(follow)
session.flush()
for playlist in playlists:
session.add(playlist)
session.flush()
for save in saves:
session.add(save)
session.flush()
for balance in balances:
session.add(balance)
session.flush()
# Refresh the lexeme matview
_update_aggregate_track(session)
session.execute("REFRESH MATERIALIZED VIEW track_lexeme_dict;")
session.execute(
UPDATE_AGGREGATE_USER_QUERY, {"prev_indexed_aggregate_block": 0}
)
session.execute("REFRESH MATERIALIZED VIEW user_lexeme_dict;")
session.execute("REFRESH MATERIALIZED VIEW aggregate_playlist;")
session.execute("REFRESH MATERIALIZED VIEW playlist_lexeme_dict;")
session.execute("REFRESH MATERIALIZED VIEW album_lexeme_dict;")
try:
output = subprocess.run(
["npm", "run", "dev"],
env=os.environ,
capture_output=True,
text=True,
cwd="es-indexer",
timeout=5,
)
raise Exception(
f"Elasticsearch indexing stopped: {output.stderr}. With env: {os.environ}"
)
except subprocess.TimeoutExpired as timeout:
if "catchup done" not in timeout.output.decode("utf-8"):
raise Exception("Elasticsearch failed to index")
def setup_search(db):
# Import app so that it'll run migrations against the db
now = datetime.now()
blocks = [
Block(
blockhash=hex(1),
number=1,
parenthash='0x01',
is_current=False,
),
Block(
blockhash=hex(2),
number=2,
parenthash='0x02',
is_current=False,
),
Block(
blockhash=hex(3),
number=3,
parenthash='0x03',
is_current=True,
)
]
tracks = [
Track(blockhash=hex(1),
blocknumber=1,
track_id=1,
is_current=True,
is_delete=False,
owner_id=1,
route_id='',
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="the track 1",
download={
"cid": None,
"is_downloadable": False,
"requires_follow": False
}),
Track(blockhash=hex(2),
blocknumber=2,
track_id=2,
is_current=True,
is_delete=False,
owner_id=2,
route_id='',
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="the track 2",
download={
"cid": None,
"is_downloadable": True,
"requires_follow": False
}),
Track(blockhash=hex(3),
blocknumber=3,
track_id=3,
is_current=True,
is_delete=False,
owner_id=1,
route_id='',
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="xyz",
download={
"cid": None,
"is_downloadable": True,
"requires_follow": False
})
]
# need users for the lexeme dict to work
users = [
User(blockhash=hex(1),
blocknumber=1,
user_id=1,
is_current=True,
handle="",
wallet="",
updated_at=now,
created_at=now),
User(blockhash=hex(2),
blocknumber=2,
user_id=2,
is_current=True,
handle="",
wallet="",
updated_at=now,
created_at=now),
User(blockhash=hex(3),
blocknumber=3,
user_id=3,
is_current=True,
handle="",
wallet="",
updated_at=now,
created_at=now)
]
with db.scoped_session() as session:
for block in blocks:
session.add(block)
session.flush()
for track in tracks:
session.add(track)
for user in users:
session.add(user)
session.flush()
# Refresh the lexeme matview
session.execute("REFRESH MATERIALIZED VIEW aggregate_track;")
session.execute("REFRESH MATERIALIZED VIEW track_lexeme_dict;")
def test_track_upload_challenge(app):
redis_conn = redis.Redis.from_url(url=REDIS_URL)
# create user
with app.app_context():
db = get_db()
today = date.today()
block1 = Block(blockhash="0x1", number=1)
block2 = Block(blockhash="0x2", number=30000000)
block3 = Block(blockhash="0x3", number=30000001)
user = User(
blockhash="0x1",
blocknumber=1,
txhash="xyz",
user_id=1,
handle="TestHandle",
handle_lc="testhandle",
is_current=True,
created_at=today - timedelta(days=100),
updated_at=today - timedelta(days=100),
)
track1 = Track(
blockhash="0x1",
blocknumber=1,
txhash="xyz",
owner_id=1,
track_id=1,
route_id="1",
track_segments=[],
is_unlisted=False,
is_current=True,
is_delete=False,
created_at=today - timedelta(days=100),
updated_at=today - timedelta(days=100),
)
track2 = Track(
blockhash="0x2",
blocknumber=30000000,
txhash="yzx",
owner_id=1,
track_id=2,
route_id="2",
track_segments=[],
is_unlisted=False,
is_current=True,
is_delete=False,
created_at=today - timedelta(days=1),
updated_at=today - timedelta(days=1),
)
track3 = Track(
blockhash="0x3",
blocknumber=30000001,
txhash="zxy",
owner_id=1,
track_id=3,
route_id="3",
track_segments=[],
is_unlisted=False,
is_current=True,
is_delete=False,
created_at=today,
updated_at=today,
)
track4 = Track(
blockhash="0x3",
blocknumber=30000001,
txhash="abc",
owner_id=1,
track_id=4,
route_id="4",
track_segments=[],
is_unlisted=False,
is_current=True,
is_delete=False,
created_at=today,
updated_at=today,
)
unlisted_track = Track(
blockhash="0x3",
blocknumber=30000001,
txhash="cba",
owner_id=1,
track_id=5,
route_id="5",
track_segments=[],
is_unlisted=True,
is_current=True,
is_delete=False,
created_at=today,
updated_at=today,
)
stem = Track(
blockhash="0x3",
blocknumber=30000001,
txhash="stem",
owner_id=1,
track_id=6,
route_id="6",
track_segments=[],
is_unlisted=False,
is_current=True,
is_delete=False,
created_at=today,
updated_at=today,
stem_of={"parent_track_id": 4, "category": "bass"},
)
with db.scoped_session() as session:
bus = ChallengeEventBus(redis_conn)
# Register events with the bus
bus.register_listener(
ChallengeEvent.track_upload, track_upload_challenge_manager
)
# set challenge as active for purposes of test
session.query(Challenge).filter(Challenge.id == "track-upload").update(
{"active": True}
)
session.add(block1)
session.add(block2)
session.add(block3)
session.flush()
session.add(user)
session.add(track1)
# Process dummy event at block number before this challenge is added
bus.dispatch(ChallengeEvent.track_upload, 1, 1)
bus.flush()
bus.process_events(session)
user_challenges = track_upload_challenge_manager.get_user_challenge_state(
session, ["1"]
)
# We should not have registered a count for this event
assert not user_challenges
# Process dummy event at block number when challenge is added
session.add(track2)
bus.dispatch(ChallengeEvent.track_upload, 30000000, 1)
bus.flush()
bus.process_events(session)
user_challenge = track_upload_challenge_manager.get_user_challenge_state(
session, ["1"]
)[0]
# We should have completed a single step (one track upload)
assert user_challenge.current_step_count == 1
assert not user_challenge.is_complete
# Ensure unlisted tracks and stems are not counted
session.add(unlisted_track)
bus.dispatch(ChallengeEvent.track_upload, 30000001, 1)
session.add(stem)
bus.dispatch(ChallengeEvent.track_upload, 30000001, 1)
bus.flush()
bus.process_events(session)
user_challenge = track_upload_challenge_manager.get_user_challenge_state(
session, ["1"]
)[0]
# Ensure stem is not counted
assert user_challenge.current_step_count == 1
# Process two more dummy events to reach the step count (i.e. 3) for completion
session.add(track3)
bus.dispatch(ChallengeEvent.track_upload, 30000001, 1)
session.add(track4)
bus.dispatch(ChallengeEvent.track_upload, 30000001, 1)
bus.flush()
bus.process_events(session)
user_challenge = track_upload_challenge_manager.get_user_challenge_state(
session, ["1"]
)[0]
# We should have completed the challenge
assert user_challenge.current_step_count == 3
assert user_challenge.is_complete
# ensure that if we lose some data now that the thing is complete, we don't change the status of the challenge
session.query(Track).filter(Track.owner_id == user.user_id).update(
{"is_delete": True}
)
session.flush()
bus.dispatch(ChallengeEvent.track_upload, 3, 1)
bus.flush()
bus.process_events(session)
user_challenge = track_upload_challenge_manager.get_user_challenge_state(
session, ["1"]
)[0]
# The challenge should still be completed
assert user_challenge.current_step_count == 3
assert user_challenge.is_complete