def setup_verified_test(session):
# Setup
blocks = [
Block(blockhash="0x1", number=1, parenthash="", is_current=False),
Block(blockhash="0x2", number=2, parenthash="", is_current=True),
]
users = [
User(
blockhash="0x1",
blocknumber=1,
user_id=1,
is_current=True,
wallet="0xFakeWallet1",
created_at=datetime.now(),
updated_at=datetime.now(),
is_verified=False,
),
User(
blockhash="0x2",
blocknumber=2,
user_id=2,
is_current=True,
wallet="0xFakeWallet2",
created_at=datetime.now(),
updated_at=datetime.now(),
is_verified=True,
),
]
challenges = [
Challenge(
id="referrals",
type=ChallengeType.aggregate,
active=True,
amount="1",
step_count=5,
),
Challenge(
id="ref-v",
type=ChallengeType.aggregate,
active=True,
amount="1",
step_count=500,
),
]
# Wipe any existing challenges in the DB from running migrations, etc
session.query(Challenge).delete()
session.commit()
session.add_all(blocks)
session.commit()
session.add_all(users)
session.add_all(challenges)
session.commit()
redis_conn = redis.Redis.from_url(url=REDIS_URL)
bus = ChallengeEventBus(redis_conn)
bus.register_listener(DEFAULT_EVENT, referral_challenge_manager)
bus.register_listener(DEFAULT_EVENT, verified_referral_challenge_manager)
return bus
def initialize_blocks_table_if_necessary(db):
target_blockhash = None
target_blockhash = update_task.shared_config["discprov"]["start_block"]
target_block = update_task.web3.eth.getBlock(target_blockhash, True)
with db.scoped_session() as session:
current_block_query_result = session.query(Block).filter_by(is_current=True)
if current_block_query_result.count() == 0:
blocks_query_result = session.query(Block)
assert (
blocks_query_result.count() == 0
), "Corrupted DB State - Expect single row marked as current"
block_model = Block(
blockhash=target_blockhash,
number=target_block.number,
parenthash=target_blockhash,
is_current=True,
)
if target_block.number == 0:
block_model.number = None
session.add(block_model)
logger.info(f"index.py | initialize_blocks_table_if_necessary | Initializing blocks table - {block_model}")
else:
assert (
current_block_query_result.count() == 1
), "Expected SINGLE row marked as current"
return target_blockhash
def setup_challenges(session):
block = Block(blockhash="0x1", number=BLOCK_NUMBER)
user = User(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
txhash="xyz",
user_id=1,
is_current=True,
handle="TestHandle",
handle_lc="testhandle",
wallet="0x1",
is_creator=False,
is_verified=False,
name="test_name",
created_at=datetime.now(),
updated_at=datetime.now(),
)
session.add(block)
session.flush()
session.add(user)
session.flush()
session.query(Challenge).filter(Challenge.id == "listen-streak").update({
"active":
True,
"starting_block":
BLOCK_NUMBER
})
def test_get_health_challenge_events_max_drift(web3_mock, redis_mock, db_mock):
"""Tests that the health check honors an unhealthy challenge events drift"""
# Set up web3 eth
def get_block(_u1, _u2): # unused
block = MagicMock()
block.number = 50
block.hash = HexBytes(b"\x50")
return block
cache_play_health_vars(redis_mock)
web3_mock.eth.get_block = get_block
# Set up redis state
redis_mock.set(challenges_last_processed_event_redis_key, int(time() - 50))
# Set up db state
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash="0x01",
number=1,
parenthash="0x01",
is_current=True,
))
args = {"challenge_events_age_max_drift": 49}
health_results, error = get_health(args)
assert error == True
assert health_results["challenge_last_event_age_sec"] < int(time() - 49)
def test_resolve_user_url(app):
"""Tests that it resolves a user url"""
with app.test_request_context():
db = get_db()
with db.scoped_session() as session:
session.add(
Block(
blockhash=
"0x2969e88561fac17ca19c1749cb3e614211ba15c8e471be55de47d0b8ca6acf5f",
parenthash="0x0000000000000000000000000000000000000000",
number=16914541,
is_current=True,
))
session.flush()
session.add(
User(
blockhash=
"0x2969e88561fac17ca19c1749cb3e614211ba15c8e471be55de47d0b8ca6acf5f",
is_current=True,
updated_at=datetime.now(),
created_at=datetime.now(),
blocknumber=16914541,
handle="Urbanbankai",
handle_lc="urbanbankai",
user_id=42727,
primary_id=1,
secondary_ids=[2, 3],
))
url = "https://audius.co/urbanbankai"
resolved_url = resolve_url(session, url)
assert resolved_url == "/v1/users/DE677"
def populate_mock_db(db, test_tracks, date):
"""Helper function to populate thee mock DB with plays"""
with db.scoped_session() as session:
for i, track_meta in enumerate(test_tracks):
blockhash = hex(i)
block = Block(
blockhash=blockhash,
number=i,
parenthash="0x01",
is_current=(i == 0),
)
track = Track(
blockhash=hex(i),
blocknumber=i,
track_id=i,
is_current=track_meta.get("is_current", True),
is_delete=track_meta.get("is_delete", False),
owner_id=300,
route_id="",
track_segments=[],
genre=track_meta.get("genre", ""),
updated_at=track_meta.get("updated_at", date),
created_at=track_meta.get("created_at", date),
is_unlisted=track_meta.get("is_unlisted", False),
)
# add block and then flush before
# adding track, bc track.blocknumber foreign key
# references block
session.add(block)
session.flush()
session.add(track)
def test_get_block_confirmation(web3_mock, redis_mock, db_mock):
"""Tests confirmation of block given a blockhash and a blocknumber"""
# Set up db state
blockhash, blocknumber = "0x01", 1
latest_blockhash, latest_blocknumber = "0x02", 2
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash="0x00",
number=0,
parenthash=None,
is_current=False,
))
session.add(
Block(
blockhash=blockhash,
number=blocknumber,
parenthash="0x00",
is_current=False,
))
session.add(
Block(
blockhash=latest_blockhash,
number=latest_blocknumber,
parenthash=blockhash,
is_current=True,
))
block_confirmation = get_block_confirmation(blockhash, blocknumber)
assert block_confirmation["block_found"] == True
assert block_confirmation["block_passed"] == True
latest_block_confirmation = get_block_confirmation(latest_blockhash,
latest_blocknumber)
assert latest_block_confirmation["block_found"] == True
assert latest_block_confirmation["block_passed"] == True
new_block_confirmation = get_block_confirmation("0xfe", 2)
assert new_block_confirmation["block_found"] == False
assert new_block_confirmation["block_passed"] == True
new_block_confirmation = get_block_confirmation("0xff", 3)
assert new_block_confirmation["block_found"] == False
assert new_block_confirmation["block_passed"] == False
def test_first_tip_challenge(app):
redis_conn = redis.Redis.from_url(url=REDIS_URL)
with app.app_context():
db = get_db()
block = Block(blockhash="0x1", number=BLOCK_NUMBER)
user = User(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
txhash="xyz",
user_id=1,
is_current=True,
handle="TestHandle",
handle_lc="testhandle",
wallet="0x1",
is_creator=False,
is_verified=False,
name="test_name",
created_at=datetime.now(),
updated_at=datetime.now(),
)
with db.scoped_session() as session:
bus = ChallengeEventBus(redis_conn)
session.query(Challenge).filter(
Challenge.id == "send-first-tip").update({
"active":
True,
"starting_block":
BLOCK_NUMBER
})
# Register events with the bus
bus.register_listener(ChallengeEvent.send_tip,
send_first_tip_challenge_manager)
session.add(block)
session.flush()
session.add(user)
session.flush()
bus.dispatch(
ChallengeEvent.send_tip,
BLOCK_NUMBER,
1, # user_id
{},
)
bus.flush()
bus.process_events(session)
session.flush()
state = send_first_tip_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
assert state.is_complete
def test_get_health_with_monitors(web3_mock, redis_mock, db_mock,
get_monitors_mock):
"""Tests that the health check returns monitor data"""
get_monitors_mock.return_value = {
"database_connections": 2,
"filesystem_size": 62725623808,
"filesystem_used": 50381168640,
"received_bytes_per_sec": 7942.038197103973,
"total_memory": 6237151232,
"used_memory": 3055149056,
"transferred_bytes_per_sec": 7340.780857447676,
}
# Set up web3 eth
def get_block(_u1, _u2): # unused
block = MagicMock()
block.number = 2
block.hash = HexBytes(b"\x02")
return block
web3_mock.eth.get_block = get_block
cache_play_health_vars(redis_mock)
# Set up db state
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash="0x01",
number=1,
parenthash="0x01",
is_current=True,
))
args = {}
health_results, error = get_health(args)
assert error == False
assert health_results["database_connections"] == 2
assert health_results["filesystem_size"] == 62725623808
assert health_results["filesystem_used"] == 50381168640
assert health_results["received_bytes_per_sec"] == 7942.038197103973
assert health_results["total_memory"] == 6237151232
assert health_results["used_memory"] == 3055149056
assert health_results["transferred_bytes_per_sec"] == 7340.780857447676
assert health_results["number_of_cpus"] == os.cpu_count()
def test_get_health_with_monitors(web3_mock, redis_mock, db_mock,
get_monitors_mock):
"""Tests that the health check returns monitor data"""
get_monitors_mock.return_value = {
'database_connections': 2,
'filesystem_size': 62725623808,
'filesystem_used': 50381168640,
'received_bytes_per_sec': 7942.038197103973,
'total_memory': 6237151232,
'used_memory': 3055149056,
'transferred_bytes_per_sec': 7340.780857447676
}
# Set up web3 eth
def getBlock(_u1, _u2): # unused
block = MagicMock()
block.number = 2
block.hash = HexBytes(b"\x02")
return block
web3_mock.eth.getBlock = getBlock
# Set up db state
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash='0x01',
number=1,
parenthash='0x01',
is_current=True,
))
args = {}
health_results, error = get_health(args)
assert error == False
assert health_results['database_connections'] == 2
assert health_results['filesystem_size'] == 62725623808
assert health_results['filesystem_used'] == 50381168640
assert health_results['received_bytes_per_sec'] == 7942.038197103973
assert health_results['total_memory'] == 6237151232
assert health_results['used_memory'] == 3055149056
assert health_results['transferred_bytes_per_sec'] == 7340.780857447676
assert health_results['number_of_cpus'] == os.cpu_count()
def test_get_health_skip_redis(web3_mock, redis_mock, db_mock):
"""Tests that the health check skips returnning redis data first if explicitly disabled"""
# Set up web3 eth
def get_block(_u1, _u2): # unused
block = MagicMock()
block.number = 2
block.hash = HexBytes(b"\x02")
return block
cache_play_health_vars(redis_mock)
web3_mock.eth.get_block = get_block
# Set up redis state
redis_mock.set(latest_block_redis_key, "3")
redis_mock.set(latest_block_hash_redis_key, "0x3")
redis_mock.set(most_recent_indexed_block_redis_key, "2")
redis_mock.set(most_recent_indexed_block_hash_redis_key, "0x02")
# Set up db state
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash="0x01",
number=1,
parenthash="0x01",
is_current=True,
))
args = {}
health_results, error = get_health(args, use_redis_cache=False)
assert error == False
assert health_results["web"]["blocknumber"] == 2
assert health_results["web"]["blockhash"] == "0x02"
assert health_results["db"]["number"] == 1
assert health_results["db"]["blockhash"] == "0x01"
assert health_results["block_difference"] == 1
assert "maximum_healthy_block_difference" in health_results
assert "version" in health_results
assert "service" in health_results
def test_get_health_using_redis(web3_mock, redis_mock, db_mock):
"""Tests that the health check returns redis data first"""
# Set up web3 eth
def getBlock(_u1, _u2): # unused
block = MagicMock()
block.number = 2
block.hash = HexBytes(b"\x02")
return block
web3_mock.eth.getBlock = getBlock
# Set up redis state
redis_mock.set(latest_block_redis_key, '3')
redis_mock.set(latest_block_hash_redis_key, '0x3')
redis_mock.set(most_recent_indexed_block_redis_key, '2')
redis_mock.set(most_recent_indexed_block_hash_redis_key, '0x02')
# Set up db state
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash='0x01',
number=1,
parenthash='0x01',
is_current=True,
))
args = {}
health_results, error = get_health(args)
assert error == False
assert health_results["web"]["blocknumber"] == 3
assert health_results["web"]["blockhash"] == "0x3"
assert health_results["db"]["number"] == 2
assert health_results["db"]["blockhash"] == "0x02"
assert health_results["block_difference"] == 1
assert "maximum_healthy_block_difference" in health_results
assert "version" in health_results
assert "service" in health_results
def setup_trending(db):
# pylint: disable=W0621
with db.scoped_session() as session:
# seed tracks + blocks
for i, track_meta in enumerate(test_tracks):
blockhash = hex(i)
block = Block(
blockhash=blockhash,
number=i,
parenthash='0x01',
is_current=True,
)
track = Track(blockhash=blockhash,
blocknumber=i,
track_id=i,
is_current=track_meta.get("is_current", True),
is_delete=track_meta.get("is_delete", False),
owner_id=300,
route_id='',
track_segments=[],
genre=track_meta.get("genre", ""),
updated_at=track_meta.get("updated_at",
datetime.now()),
created_at=track_meta.get("created_at",
datetime.now()),
is_unlisted=track_meta.get("is_unlisted", False))
# add block and then flush before
# adding track, bc track.blocknumber foreign key
# references block
session.add(block)
session.flush()
session.add(track)
# seed plays
for i, play_meta in enumerate(test_plays):
play = Play(id=i,
play_item_id=play_meta.get("item_id"),
created_at=play_meta.get("created_at", datetime.now()))
session.add(play)
def test_get_health(web3_mock, redis_mock, db_mock):
"""Tests that the health check returns db data"""
# Set up web3 eth
def get_block(_u1, _u2): # unused
block = MagicMock()
block.number = 2
block.hash = HexBytes(b"\x02")
return block
cache_play_health_vars(redis_mock)
web3_mock.eth.get_block = get_block
# Set up db state
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash="0x01",
number=1,
parenthash="0x01",
is_current=True,
))
args = {}
health_results, error = get_health(args)
assert error == False
assert health_results["web"]["blocknumber"] == 2
assert health_results["web"]["blockhash"] == "0x02"
assert health_results["db"]["number"] == 1
assert health_results["db"]["blockhash"] == "0x01"
assert health_results["block_difference"] == 1
assert health_results["plays"]["tx_info"]["slot_diff"] == 3
assert "maximum_healthy_block_difference" in health_results
assert "version" in health_results
assert "service" in health_results
def test_get_health_unhealthy_block_difference(web3_mock, redis_mock, db_mock):
"""Tests that the health check an unhealthy block difference"""
# Set up web3 eth
def get_block(_u1, _u2): # unused
block = MagicMock()
block.number = 50
block.hash = HexBytes(b"\x50")
return block
cache_play_health_vars(redis_mock)
web3_mock.eth.get_block = get_block
# Set up db state
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash="0x01",
number=1,
parenthash="0x01",
is_current=True,
))
args = {"enforce_block_diff": True, "healthy_block_diff": 40}
health_results, error = get_health(args)
assert error == True
assert health_results["web"]["blocknumber"] == 50
assert health_results["web"]["blockhash"] == "0x50"
assert health_results["db"]["number"] == 1
assert health_results["db"]["blockhash"] == "0x01"
assert health_results["block_difference"] == 49
assert "maximum_healthy_block_difference" in health_results
assert "version" in health_results
assert "service" in health_results
def test_get_health_with_invalid_db_state(web3_mock, redis_mock, db_mock):
"""Tests that the health check can handle an invalid block in the db"""
# Set up web3 eth
def getBlock(_u1, _u2): # unused
block = MagicMock()
block.number = 2
block.hash = HexBytes(b"\x02")
return block
web3_mock.eth.getBlock = getBlock
# Set up db state
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash='0x01',
number=None, # NoneType
parenthash='0x01',
is_current=True,
))
args = {}
health_results, error = get_health(args)
assert error == False
assert health_results["web"]["blocknumber"] == 2
assert health_results["web"]["blockhash"] == "0x02"
assert health_results["db"]["number"] == 0
assert health_results["db"]["blockhash"] == "0x01"
assert health_results["block_difference"] == 2
assert "maximum_healthy_block_difference" in health_results
assert "version" in health_results
assert "service" in health_results
def index_blocks(self, db, blocks_list):
web3 = update_task.web3
redis = update_task.redis
num_blocks = len(blocks_list)
block_order_range = range(len(blocks_list) - 1, -1, -1)
for i in block_order_range:
block = blocks_list[i]
block_index = num_blocks - i
block_number = block.number
block_timestamp = block.timestamp
logger.info(
f"index.py | index_blocks | {self.request.id} | block {block.number} - {block_index}/{num_blocks}"
)
# Handle each block in a distinct transaction
with db.scoped_session() as session:
current_block_query = session.query(Block).filter_by(
is_current=True)
# Without this check we may end up duplicating an insert operation
block_model = Block(
blockhash=web3.toHex(block.hash),
parenthash=web3.toHex(block.parentHash),
number=block.number,
is_current=True,
)
# Update blocks table after
assert (current_block_query.count() == 1
), "Expected single row marked as current"
former_current_block = current_block_query.first()
former_current_block.is_current = False
session.add(block_model)
user_factory_txs = []
track_factory_txs = []
social_feature_factory_txs = []
playlist_factory_txs = []
user_library_factory_txs = []
tx_receipt_dict = fetch_tx_receipts(self, block.transactions)
# Sort transactions by hash
sorted_txs = sorted(block.transactions,
key=lambda entry: entry['hash'])
# Parse tx events in each block
for tx in sorted_txs:
tx_hash = web3.toHex(tx["hash"])
tx_target_contract_address = tx["to"]
tx_receipt = tx_receipt_dict[tx_hash]
# Handle user operations
if tx_target_contract_address == contract_addresses[
"user_factory"]:
logger.info(
f"index.py | index_blocks | UserFactory contract addr: {tx_target_contract_address}"
f" tx from block - {tx}, receipt - {tx_receipt}, adding to user_factory_txs to process in bulk"
)
user_factory_txs.append(tx_receipt)
# Handle track operations
if tx_target_contract_address == contract_addresses[
"track_factory"]:
logger.info(
f"index.py | index_blocks | TrackFactory contract addr: {tx_target_contract_address}"
f" tx from block - {tx}, receipt - {tx_receipt}")
# Track state operations
track_factory_txs.append(tx_receipt)
# Handle social operations
if tx_target_contract_address == contract_addresses[
"social_feature_factory"]:
logger.info(
f"index.py | index_blocks | Social feature contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}")
social_feature_factory_txs.append(tx_receipt)
# Handle repost operations
if tx_target_contract_address == contract_addresses[
"playlist_factory"]:
logger.info(
f"index.py | index_blocks | Playlist contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}")
playlist_factory_txs.append(tx_receipt)
# Handle User Library operations
if tx_target_contract_address == contract_addresses[
"user_library_factory"]:
logger.info(
f"index.py | index_blocks | User Library contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}")
user_library_factory_txs.append(tx_receipt)
# bulk process operations once all tx's for block have been parsed
total_user_changes, user_ids = user_state_update(
self, update_task, session, user_factory_txs, block_number,
block_timestamp)
user_state_changed = total_user_changes > 0
total_track_changes, track_ids = track_state_update(
self, update_task, session, track_factory_txs, block_number,
block_timestamp)
track_state_changed = total_track_changes > 0
social_feature_state_changed = ( # pylint: disable=W0612
social_feature_state_update(self, update_task, session,
social_feature_factory_txs,
block_number, block_timestamp) > 0)
# Playlist state operations processed in bulk
total_playlist_changes, playlist_ids = playlist_state_update(
self, update_task, session, playlist_factory_txs, block_number,
block_timestamp)
playlist_state_changed = total_playlist_changes > 0
user_library_state_changed = user_library_state_update( # pylint: disable=W0612
self, update_task, session, user_library_factory_txs,
block_number, block_timestamp)
track_lexeme_state_changed = (user_state_changed
or track_state_changed)
session.commit()
if user_state_changed:
if user_ids:
remove_cached_user_ids(redis, user_ids)
if track_lexeme_state_changed:
if track_ids:
remove_cached_track_ids(redis, track_ids)
if playlist_state_changed:
if playlist_ids:
remove_cached_playlist_ids(redis, playlist_ids)
# add the block number of the most recently processed block to redis
redis.set(most_recent_indexed_block_redis_key, block.number)
redis.set(most_recent_indexed_block_hash_redis_key, block.hash.hex())
if num_blocks > 0:
logger.warning(
f"index.py | index_blocks | Indexed {num_blocks} blocks")
def test_profile_completion_challenge_with_playlists(app):
redis_conn = redis.Redis.from_url(url=REDIS_URL)
# create user
with app.app_context():
db = get_db()
block = Block(blockhash="0x1", number=BLOCK_NUMBER)
user = User(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
txhash="xyz",
user_id=1,
is_current=True,
handle="TestHandle",
handle_lc="testhandle",
wallet="0x123",
is_creator=False,
is_verified=False,
name="test_name",
created_at=datetime.now(),
updated_at=datetime.now(),
)
with db.scoped_session() as session:
bus = ChallengeEventBus(redis_conn)
# set challenge as active for purposes of test
session.query(Challenge).filter(
Challenge.id == "profile-completion").update({
"active":
True,
"starting_block":
BLOCK_NUMBER
})
# Register events with the bus
bus.register_listener(ChallengeEvent.profile_update,
profile_challenge_manager)
bus.register_listener(ChallengeEvent.repost, profile_challenge_manager)
bus.register_listener(ChallengeEvent.follow, profile_challenge_manager)
bus.register_listener(ChallengeEvent.favorite,
profile_challenge_manager)
session.add(block)
session.flush()
session.add(user)
# Process dummy event just to get this thing initted
bus.dispatch(ChallengeEvent.profile_update, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
# We should have completed a single step (name)
assert state.current_step_count == 1 and not state.is_complete
# Do a repost
repost = Repost(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
user_id=1,
repost_item_id=1,
repost_type=RepostType.playlist,
is_current=True,
is_delete=False,
created_at=datetime.now(),
)
session.add(repost)
session.flush()
bus.dispatch(ChallengeEvent.repost, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
assert state.current_step_count == 2 and not state.is_complete
# Do a save
save = Save(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
user_id=1,
save_item_id=1,
save_type=SaveType.playlist,
is_current=True,
is_delete=False,
created_at=datetime.now(),
)
session.add(save)
session.flush()
bus.dispatch(ChallengeEvent.favorite, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
session.flush()
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
assert state.current_step_count == 3 and not state.is_complete
# Do 1 follow, then 5 total follows
follow = Follow(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
is_current=True,
is_delete=False,
created_at=datetime.now(),
follower_user_id=1,
followee_user_id=2,
)
session.add(follow)
session.flush()
bus.dispatch(ChallengeEvent.follow, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
session.flush()
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
# Assert 1 follow didn't do anything
assert state.current_step_count == 3 and not state.is_complete
follows = [
Follow(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
is_current=True,
is_delete=False,
created_at=datetime.now(),
follower_user_id=1,
followee_user_id=3,
),
Follow(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
is_current=True,
is_delete=False,
created_at=datetime.now(),
follower_user_id=1,
followee_user_id=4,
),
Follow(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
is_current=True,
is_delete=False,
created_at=datetime.now(),
follower_user_id=1,
followee_user_id=5,
),
Follow(
blockhash="0x1",
blocknumber=BLOCK_NUMBER,
is_current=True,
is_delete=False,
created_at=datetime.now(),
follower_user_id=1,
followee_user_id=6,
),
]
session.add_all(follows)
session.flush()
bus.dispatch(ChallengeEvent.follow, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
assert state.current_step_count == 4 and not state.is_complete
# profile_picture
session.query(User).filter(User.user_id == 1).update(
{"profile_picture": "profilepictureurl"})
session.flush()
bus.dispatch(ChallengeEvent.profile_update, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
assert state.current_step_count == 5 and not state.is_complete
# profile description
session.query(User).filter(User.user_id == 1).update(
{"bio": "profiledescription"})
session.flush()
bus.dispatch(ChallengeEvent.profile_update, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
assert state.current_step_count == 6 and not state.is_complete
# Undo it, ensure that our count goes down
session.query(User).filter(User.user_id == 1).update({"bio": None})
session.flush()
bus.dispatch(ChallengeEvent.profile_update, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
assert state.current_step_count == 5 and not state.is_complete
# profile_cover_photo
session.query(User).filter(User.user_id == 1).update({
"bio":
"profiledescription",
"cover_photo":
"test_cover_photo"
})
session.flush()
bus.dispatch(ChallengeEvent.profile_update, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
assert state.current_step_count == 7 and state.is_complete == True
# ensure that if we lose some data now that the thing is complete, we don't change the status of the challenge
session.query(User).filter(User.user_id == 1).update(
{"cover_photo": None})
session.flush()
bus.dispatch(ChallengeEvent.profile_update, BLOCK_NUMBER, 1)
bus.flush()
bus.process_events(session)
state = profile_challenge_manager.get_user_challenge_state(
session, ["1"])[0]
assert state.current_step_count == 7 and state.is_complete == True
def setup_extra_metadata_test(session):
blocks = [Block(blockhash="0x1", number=1, parenthash="", is_current=True)]
users = [
User(
blockhash="0x1",
blocknumber=1,
user_id=1,
is_current=True,
wallet="0x38C68fF3926bf4E68289672F75ee1543117dD9B3",
created_at=datetime.now(),
updated_at=datetime.now(),
)
]
challenges = [
# Test numeric challenges
# Numeric 1 with default extra data, no completion
Challenge(
id="numeric_1",
type=ChallengeType.numeric,
active=True,
amount="5",
step_count=5,
),
# Numeric 2 with some extra data
Challenge(
id="numeric_2",
type=ChallengeType.numeric,
active=True,
amount="5",
step_count=5,
),
]
user_challenges = [
UserChallenge(
challenge_id="numeric_2",
user_id=1,
specifier="1",
is_complete=False,
current_step_count=5,
),
]
session.query(Challenge).delete()
session.commit()
session.add_all(blocks)
session.commit()
session.add_all(users)
session.commit()
session.add_all(challenges)
session.commit()
session.add_all(user_challenges)
session.commit()
redis_conn = redis.Redis.from_url(url=REDIS_URL)
bus = ChallengeEventBus(redis_conn)
bus.register_listener(
DEFAULT_EVENT, ChallengeManager("numeric_1", NumericCustomUpdater()))
bus.register_listener(
DEFAULT_EVENT, ChallengeManager("numeric_2", NumericCustomUpdater()))
return bus
def test_user_indexing_skip_tx(bus_mock: mock.MagicMock, app, mocker):
"""Tests that users skip cursed txs without throwing an error and are able to process other tx in block"""
with app.app_context():
db = get_db()
redis = get_redis()
web3 = Web3()
bus_mock(redis)
update_task = DatabaseTask(
cid_metadata_client=cid_metadata_client,
web3=web3,
challenge_event_bus=bus_mock,
redis=redis,
)
class TestUserTransaction:
def __init__(self):
self.transactionHash = None
pass
blessed_tx_hash = (
"0x34004dfaf5bb7cf9998eaf387b877d72d198c6508608e309df3f89e57def4db3")
blessed_tx = TestUserTransaction()
blessed_tx.transactionHash = update_task.web3.toBytes(
hexstr=blessed_tx_hash)
cursed_tx_hash = (
"0x5fe51d735309d3044ae30055ad29101018a1a399066f6c53ea23800225e3a3be")
cursed_tx = TestUserTransaction()
cursed_tx.transactionHash = update_task.web3.toBytes(hexstr=cursed_tx_hash)
test_block_number = 25278765
test_block_timestamp = 1
test_block_hash = update_task.web3.toHex(block_hash)
test_user_factory_txs = [cursed_tx, blessed_tx]
test_timestamp = datetime.utcfromtimestamp(test_block_timestamp)
blessed_user_record = User(
blockhash=test_block_hash,
blocknumber=test_block_number,
txhash=blessed_tx_hash,
user_id=91232,
name="tobey maguire",
is_creator=False,
is_current=True,
updated_at=test_timestamp,
created_at=test_timestamp,
)
cursed_user_record = User(
blockhash=test_block_hash,
blocknumber=test_block_number,
txhash=cursed_tx_hash,
user_id=91238,
name="birbs",
is_current=None,
is_creator=None,
updated_at=test_timestamp,
created_at=None,
)
mocker.patch(
"src.tasks.users.lookup_user_record",
side_effect=[cursed_user_record, blessed_user_record],
autospec=True,
)
mocker.patch(
"src.tasks.users.get_user_events_tx",
side_effect=[
[], # no user added events
[],
[
{
"args":
AttrDict({
"_userId": cursed_user_record.user_id,
"_name": cursed_user_record.name.encode("utf-8"),
})
},
], # update name event
[],
[],
[],
[],
[],
[],
[],
[], # second tx receipt
[],
[
{
"args":
AttrDict({
"_userId": blessed_user_record.user_id,
"_name": blessed_user_record.name.encode("utf-8"),
})
},
], # update name event
[],
[],
[],
[],
[],
[],
[],
],
autospec=True,
)
test_ipfs_metadata: Dict[str, Any] = {}
test_blacklisted_cids: Dict[str, Any] = {}
with db.scoped_session() as session, bus_mock.use_scoped_dispatch_queue():
try:
current_block = Block(
blockhash=test_block_hash,
parenthash=test_block_hash,
number=test_block_number,
is_current=True,
)
session.add(current_block)
(total_changes, updated_user_ids_set) = user_state_update(
update_task,
update_task,
session,
test_user_factory_txs,
test_block_number,
test_block_timestamp,
block_hash,
test_ipfs_metadata,
test_blacklisted_cids,
)
assert len(updated_user_ids_set) == 1
assert list(updated_user_ids_set)[0] == blessed_user_record.user_id
assert total_changes == 1
assert (session.query(SkippedTransaction).filter(
SkippedTransaction.txhash == cursed_user_record.txhash,
SkippedTransaction.level == SkippedTransactionLevel.node,
).first())
assert (session.query(User).filter(
User.user_id == blessed_user_record.user_id).first())
assert (session.query(User).filter(
User.user_id == cursed_user_record.user_id).first()) == None
except Exception:
assert False
def setup_listen_streak_challenge(session):
# Setup
blocks = [
Block(blockhash="0x1", number=1, parenthash="", is_current=False),
Block(blockhash="0x2", number=2, parenthash="", is_current=True),
]
users = [
User(
blockhash="0x1",
blocknumber=1,
user_id=1,
is_current=True,
wallet="0xFakeWallet1",
created_at=datetime.now(),
updated_at=datetime.now(),
is_verified=False,
),
User(
blockhash="0x2",
blocknumber=2,
user_id=2,
is_current=True,
wallet="0xFakeWallet2",
created_at=datetime.now(),
updated_at=datetime.now(),
is_verified=True,
),
]
challenges = [
Challenge(
id="listen-streak",
type=ChallengeType.numeric,
active=True,
amount="1",
step_count=7,
)
]
user_challenges = [
UserChallenge(
challenge_id="listen-streak",
user_id=1,
specifier="1",
is_complete=False,
current_step_count=5,
),
UserChallenge(
challenge_id="listen-streak",
user_id=2,
specifier="2",
is_complete=False,
current_step_count=5,
),
]
listen_streak_challenges = [
ListenStreakChallenge(
user_id=1,
last_listen_date=datetime.now() - timedelta(hours=12),
listen_streak=5,
),
ListenStreakChallenge(
user_id=2,
last_listen_date=datetime.now() - timedelta(hours=50),
listen_streak=5,
),
]
# Wipe any existing challenges in the DB from running migrations, etc
session.query(Challenge).delete()
session.commit()
session.add_all(blocks)
session.commit()
session.add_all(users)
session.add_all(challenges)
session.commit()
session.add_all(user_challenges)
session.commit()
session.add_all(listen_streak_challenges)
session.commit()
redis_conn = redis.Redis.from_url(url=REDIS_URL)
bus = ChallengeEventBus(redis_conn)
bus.register_listener(DEFAULT_EVENT, listen_streak_challenge_manager)
return bus
def index_blocks(self, db, blocks_list):
web3 = update_task.web3
redis = update_task.redis
num_blocks = len(blocks_list)
block_order_range = range(len(blocks_list) - 1, -1, -1)
for i in block_order_range:
if i % 10 == 0 and i != 0:
block_index = num_blocks - i
logger.info(f"index.py | index_blocks | processing block {block_index}/{num_blocks} blocks")
block = blocks_list[i]
block_number = block.number
block_timestamp = block.timestamp
# Handle each block in a distinct transaction
with db.scoped_session() as session:
current_block_query = session.query(Block).filter_by(is_current=True)
block_model = Block(
blockhash=web3.toHex(block.hash),
parenthash=web3.toHex(block.parentHash),
number=block.number,
is_current=True,
)
# Update blocks table after
assert (
current_block_query.count() == 1
), "Expected single row marked as current"
former_current_block = current_block_query.first()
former_current_block.is_current = False
session.add(block_model)
user_factory_txs = []
track_factory_txs = []
social_feature_factory_txs = []
playlist_factory_txs = []
user_library_factory_txs = []
# Sort transactions by hash
sorted_txs = sorted(block.transactions, key=lambda entry: entry['hash'])
# Parse tx events in each block
for tx in sorted_txs:
tx_hash = web3.toHex(tx["hash"])
tx_target_contract_address = tx["to"]
tx_receipt = web3.eth.getTransactionReceipt(tx_hash)
# Handle user operations
if tx_target_contract_address == contract_addresses["user_factory"]:
logger.info(
f"index.py | index_blocks | UserFactory contract addr: {tx_target_contract_address}"
f" tx from block - {tx}, receipt - {tx_receipt}, adding to user_factory_txs to process in bulk"
)
user_factory_txs.append(tx_receipt)
# Handle track operations
if tx_target_contract_address == contract_addresses["track_factory"]:
logger.info(
f"index.py | index_blocks | TrackFactory contract addr: {tx_target_contract_address}"
f" tx from block - {tx}, receipt - {tx_receipt}"
)
# Track state operations
track_factory_txs.append(tx_receipt)
# Handle social operations
if tx_target_contract_address == contract_addresses["social_feature_factory"]:
logger.info(
f"index.py | index_blocks | Social feature contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}"
)
social_feature_factory_txs.append(tx_receipt)
# Handle repost operations
if tx_target_contract_address == contract_addresses["playlist_factory"]:
logger.info(
f"index.py | index_blocks | Playlist contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}"
)
playlist_factory_txs.append(tx_receipt)
# Handle User Library operations
if tx_target_contract_address == contract_addresses["user_library_factory"]:
logger.info(
f"index.py | index_blocks | User Library contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}"
)
user_library_factory_txs.append(tx_receipt)
# bulk process operations once all tx's for block have been parsed
user_state_changed = (
user_state_update(
self, update_task, session, user_factory_txs, block_number, block_timestamp
)
> 0
)
track_state_changed = (
track_state_update(
self, update_task, session, track_factory_txs, block_number, block_timestamp
)
> 0
)
social_feature_state_changed = ( # pylint: disable=W0612
social_feature_state_update(
self, update_task, session, social_feature_factory_txs, block_number, block_timestamp
)
> 0
)
# Playlist state operations processed in bulk
playlist_state_changed = playlist_state_update(
self, update_task, session, playlist_factory_txs, block_number, block_timestamp
)
user_library_state_changed = user_library_state_update( # pylint: disable=W0612
self, update_task, session, user_library_factory_txs, block_number, block_timestamp
)
# keep search materialized view in sync with db
# only refresh track_lexeme_dict when necessary
# social state changes are not factored in since they don't affect track_lexeme_dict
# write out all pending transactions to db before refreshing view
track_lexeme_state_changed = (user_state_changed or track_state_changed)
session.flush()
if user_state_changed:
session.execute("REFRESH MATERIALIZED VIEW user_lexeme_dict")
if track_lexeme_state_changed:
session.execute("REFRESH MATERIALIZED VIEW track_lexeme_dict")
if playlist_state_changed:
session.execute("REFRESH MATERIALIZED VIEW playlist_lexeme_dict")
session.execute("REFRESH MATERIALIZED VIEW album_lexeme_dict")
# add the block number of the most recently processed block to redis
redis.set(most_recent_indexed_block_redis_key, block.number)
if num_blocks > 0:
logger.warning(f"index.py | index_blocks | Indexed {num_blocks} blocks")
def setup_db(session):
blocks = [Block(blockhash="0x1", number=1, parenthash="", is_current=True)]
users = [
User(
blockhash="0x1",
blocknumber=1,
user_id=1,
is_current=True,
wallet="0x38C68fF3926bf4E68289672F75ee1543117dD9B3",
created_at=datetime.now(),
updated_at=datetime.now(),
)
]
challenges = [
Challenge(
id="boolean_challenge_1",
type=ChallengeType.boolean,
active=True,
amount="5",
),
Challenge(
id="boolean_challenge_2",
type=ChallengeType.boolean,
active=True,
amount="5",
),
Challenge(
id="boolean_challenge_3",
type=ChallengeType.boolean,
active=True,
amount="5",
),
# No progress on this, but active
# should be returned
Challenge(
id="boolean_challenge_4",
type=ChallengeType.boolean,
active=True,
amount="5",
),
# Inactive, with no progress
Challenge(
id="boolean_challenge_5",
type=ChallengeType.boolean,
active=False,
amount="5",
),
# Inactive, WITH progress
Challenge(
id="boolean_challenge_6",
type=ChallengeType.boolean,
active=False,
amount="5",
),
Challenge(
id="trending_challenge_1",
type=ChallengeType.trending,
active=True,
amount="5",
),
Challenge(
id="aggregate_challenge_1",
type=ChallengeType.aggregate,
active=True,
amount="5",
step_count=3,
),
Challenge(
id="aggregate_challenge_2",
type=ChallengeType.aggregate,
active=True,
amount="5",
step_count=2,
),
Challenge(
id="aggregate_challenge_3",
type=ChallengeType.aggregate,
active=True,
amount="5",
step_count=2,
),
Challenge(id="trending_1",
type=ChallengeType.trending,
active=True,
amount="5"),
Challenge(id="trending_2",
type=ChallengeType.trending,
active=True,
amount="5"),
Challenge(id="trending_3",
type=ChallengeType.trending,
active=True,
amount="5"),
]
user_challenges = [
# Finished the first challenge, disbursed
UserChallenge(
challenge_id="boolean_challenge_1",
user_id=1,
specifier="1",
is_complete=True,
),
# Did finish the second challenge, did not disburse
UserChallenge(
challenge_id="boolean_challenge_2",
user_id=1,
specifier="1",
is_complete=True,
),
# Did not finish challenge 3
UserChallenge(
challenge_id="boolean_challenge_3",
user_id=1,
specifier="1",
is_complete=False,
),
# Inactive challenge
UserChallenge(
challenge_id="boolean_challenge_6",
user_id=1,
specifier="1",
is_complete=True,
),
UserChallenge(
challenge_id="aggregate_challenge_1",
user_id=1,
specifier="1-2", # compound specifiers, like if user1 invites user2
is_complete=True,
),
# Ensure that a non-complete user-challenge isn't counted towards
# aggregate challenge score
UserChallenge(
challenge_id="aggregate_challenge_1",
user_id=1,
specifier="1-3",
is_complete=False,
),
UserChallenge(
challenge_id="aggregate_challenge_2",
user_id=1,
specifier="1-2",
is_complete=True,
),
UserChallenge(
challenge_id="aggregate_challenge_2",
user_id=1,
specifier="1-3",
is_complete=True,
),
# Trending 1 should be finished and included
UserChallenge(
challenge_id="trending_1",
user_id=1,
specifier="06-01-2020",
is_complete=True,
),
# Trending 2 should not be included
UserChallenge(
challenge_id="trending_2",
user_id=1,
specifier="06-01-2020",
is_complete=False,
),
]
disbursements = [
ChallengeDisbursement(
challenge_id="boolean_challenge_1",
user_id=1,
amount="5",
signature="1",
slot=1,
specifier="1",
)
]
# Wipe any existing challenges in the DB from running migrations, etc
session.query(Challenge).delete()
session.commit()
session.add_all(blocks)
session.commit()
session.add_all(users)
session.commit()
session.add_all(challenges)
session.commit()
session.add_all(user_challenges)
session.commit()
session.add_all(disbursements)
redis_conn = redis.Redis.from_url(url=REDIS_URL)
bus = ChallengeEventBus(redis_conn)
challenge_types = [
"boolean_challenge_1",
"boolean_challenge_2",
"boolean_challenge_3",
"boolean_challenge_4",
"boolean_challenge_5",
"boolean_challenge_6",
"trending_challenge_1",
"aggregate_challenge_1",
"aggregate_challenge_2",
"aggregate_challenge_3",
"trending_1",
"trending_2",
"trending_3",
]
for ct in challenge_types:
bus.register_listener(
DEFAULT_EVENT,
ChallengeManager(ct, DefaultUpdater()),
)
return bus
def setup_search(app_module):
with app_module.app_context():
db = get_db()
# Import app so that it'll run migrations against the db
now = datetime.now()
blocks = [
Block(
blockhash=hex(1),
number=1,
parenthash="0x01",
is_current=False,
),
Block(
blockhash=hex(2),
number=2,
parenthash="0x02",
is_current=False,
),
Block(
blockhash=hex(3),
number=3,
parenthash="0x03",
is_current=True,
),
]
tracks = [
Track(
blockhash=hex(1),
blocknumber=1,
track_id=1,
is_current=True,
is_delete=False,
owner_id=1,
route_id="",
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="the track 1",
download={"cid": None, "is_downloadable": False, "requires_follow": False},
),
Track(
blockhash=hex(2),
blocknumber=2,
track_id=2,
is_current=True,
is_delete=False,
owner_id=2,
route_id="",
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="the track 2",
download={"cid": None, "is_downloadable": True, "requires_follow": False},
),
Track(
blockhash=hex(3),
blocknumber=3,
track_id=3,
is_current=True,
is_delete=False,
owner_id=1,
route_id="",
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="xyz",
download={"cid": None, "is_downloadable": True, "requires_follow": False},
),
]
# need users for the lexeme dict to work
users = [
User(
blockhash=hex(1),
blocknumber=1,
user_id=1,
is_current=True,
handle="",
wallet="",
name="user 1",
updated_at=now,
created_at=now,
),
User(
blockhash=hex(2),
blocknumber=2,
user_id=2,
is_current=True,
handle="",
name="user 2",
wallet="",
updated_at=now,
created_at=now,
),
User(
blockhash=hex(3),
blocknumber=3,
user_id=3,
is_current=True,
handle="",
wallet="",
name="fdwea",
updated_at=now,
created_at=now,
),
]
follows = [
Follow(
blockhash=hex(1),
blocknumber=1,
follower_user_id=2,
followee_user_id=1,
is_current=True,
is_delete=False,
created_at=now,
)
]
playlists = [
Playlist(
blockhash=hex(1),
blocknumber=1,
playlist_id=1,
playlist_owner_id=1,
is_album=False,
is_private=False,
playlist_name="playlist 1",
playlist_contents={"track_ids": [{"track": 1, "time": 1}]},
is_current=True,
is_delete=False,
updated_at=now,
created_at=now,
),
Playlist(
blockhash=hex(2),
blocknumber=2,
playlist_id=2,
playlist_owner_id=2,
is_album=True,
is_private=False,
playlist_name="album 1",
playlist_contents={"track_ids": [{"track": 2, "time": 2}]},
is_current=True,
is_delete=False,
updated_at=now,
created_at=now,
),
]
saves = [
Save(
blockhash=hex(1),
blocknumber=1,
user_id=1,
save_item_id=1,
save_type=SaveType.track,
created_at=now,
is_current=True,
is_delete=False,
),
Save(
blockhash=hex(1),
blocknumber=1,
user_id=1,
save_item_id=1,
save_type=SaveType.playlist,
created_at=now,
is_current=True,
is_delete=False,
),
Save(
blockhash=hex(1),
blocknumber=1,
user_id=1,
save_item_id=2,
save_type=SaveType.album,
created_at=now,
is_current=True,
is_delete=False,
),
]
balances = [
UserBalance(
user_id=1,
balance=0,
associated_wallets_balance=0,
associated_sol_wallets_balance=0,
waudio=0,
)
]
with db.scoped_session() as session:
for block in blocks:
session.add(block)
session.flush()
for track in tracks:
session.add(track)
for user in users:
session.add(user)
session.flush()
for follow in follows:
session.add(follow)
session.flush()
for playlist in playlists:
session.add(playlist)
session.flush()
for save in saves:
session.add(save)
session.flush()
for balance in balances:
session.add(balance)
session.flush()
# Refresh the lexeme matview
_update_aggregate_track(session)
session.execute("REFRESH MATERIALIZED VIEW track_lexeme_dict;")
session.execute(
UPDATE_AGGREGATE_USER_QUERY, {"prev_indexed_aggregate_block": 0}
)
session.execute("REFRESH MATERIALIZED VIEW user_lexeme_dict;")
session.execute("REFRESH MATERIALIZED VIEW aggregate_playlist;")
session.execute("REFRESH MATERIALIZED VIEW playlist_lexeme_dict;")
session.execute("REFRESH MATERIALIZED VIEW album_lexeme_dict;")
try:
output = subprocess.run(
["npm", "run", "dev"],
env=os.environ,
capture_output=True,
text=True,
cwd="es-indexer",
timeout=5,
)
raise Exception(
f"Elasticsearch indexing stopped: {output.stderr}. With env: {os.environ}"
)
except subprocess.TimeoutExpired as timeout:
if "catchup done" not in timeout.output.decode("utf-8"):
raise Exception("Elasticsearch failed to index")
def test_playlist_indexing_skip_tx(app, mocker):
"""Tests that playlists skip cursed txs without throwing an error and are able to process other tx in block"""
with app.app_context():
db = get_db()
cid_metadata_client = CIDMetadataClient({})
web3 = Web3()
challenge_event_bus = setup_challenge_bus()
update_task = UpdateTask(cid_metadata_client, web3,
challenge_event_bus)
class TestPlaylistTransaction:
pass
blessed_tx_hash = (
"0x34004dfaf5bb7cf9998eaf387b877d72d198c6508608e309df3f89e57def4db3")
blessed_tx = TestPlaylistTransaction()
blessed_tx.transactionHash = update_task.web3.toBytes(
hexstr=blessed_tx_hash)
cursed_tx_hash = (
"0x5fe51d735309d3044ae30055ad29101018a1a399066f6c53ea23800225e3a3be")
cursed_tx = TestPlaylistTransaction()
cursed_tx.transactionHash = update_task.web3.toBytes(hexstr=cursed_tx_hash)
test_block_number = 25278765
test_block_timestamp = 1
test_block_hash = update_task.web3.toHex(block_hash)
test_playlist_factory_txs = [cursed_tx, blessed_tx]
test_timestamp = datetime.utcfromtimestamp(test_block_timestamp)
blessed_playlist_record = Playlist(
blockhash=test_block_hash,
blocknumber=test_block_number,
txhash=blessed_tx_hash,
playlist_id=91232,
is_album=False,
is_private=False,
playlist_name="test",
playlist_contents={},
playlist_image_multihash=None,
playlist_image_sizes_multihash=None,
description="testing!",
upc=None,
is_current=True,
is_delete=True,
last_added_to=None,
updated_at=test_timestamp,
created_at=test_timestamp,
playlist_owner_id=1,
)
cursed_playlist_record = Playlist(
blockhash=test_block_hash,
blocknumber=test_block_number,
txhash=cursed_tx_hash,
playlist_id=91238,
is_album=None,
is_private=None,
playlist_name=None,
playlist_image_multihash=None,
playlist_image_sizes_multihash=None,
description=None,
upc=None,
is_current=True,
is_delete=True,
last_added_to=None,
updated_at=test_timestamp,
created_at=None,
)
mocker.patch(
"src.tasks.playlists.lookup_playlist_record",
side_effect=[cursed_playlist_record, blessed_playlist_record],
autospec=True,
)
mocker.patch(
"src.tasks.playlists.get_playlist_events_tx",
side_effect=[
[], # no playlist created events
[
{
"args":
AttrDict({
"_playlistId": cursed_playlist_record.playlist_id,
})
},
], # playlist deleted event
[],
[],
[],
[],
[],
[],
[],
[],
[], # second tx receipt
[
{
"args":
AttrDict({
"_playlistId":
blessed_playlist_record.playlist_id,
})
},
], # playlist deleted event
[],
[],
[],
[],
[],
[],
[],
[],
],
autospec=True,
)
test_ipfs_metadata = {}
test_blacklisted_cids = {}
with db.scoped_session() as session:
try:
current_block = Block(
blockhash=test_block_hash,
parenthash=test_block_hash,
number=test_block_number,
is_current=True,
)
session.add(current_block)
(total_changes, updated_playlist_ids_set) = playlist_state_update(
update_task,
update_task,
session,
test_playlist_factory_txs,
test_block_number,
test_block_timestamp,
block_hash,
test_ipfs_metadata,
test_blacklisted_cids,
)
assert len(updated_playlist_ids_set) == 1
assert (list(updated_playlist_ids_set)[0] ==
blessed_playlist_record.playlist_id)
assert total_changes == 1
assert (session.query(SkippedTransaction).filter(
SkippedTransaction.txhash == cursed_playlist_record.txhash,
SkippedTransaction.level == SkippedTransactionLevel.node,
).first())
assert (session.query(Playlist).filter(
Playlist.playlist_id ==
blessed_playlist_record.playlist_id).first())
assert (session.query(Playlist).filter(
Playlist.playlist_id ==
cursed_playlist_record.playlist_id).first()) == None
except Exception:
assert False
def setup_search(db):
# Import app so that it'll run migrations against the db
now = datetime.now()
blocks = [
Block(
blockhash=hex(1),
number=1,
parenthash='0x01',
is_current=False,
),
Block(
blockhash=hex(2),
number=2,
parenthash='0x02',
is_current=False,
),
Block(
blockhash=hex(3),
number=3,
parenthash='0x03',
is_current=True,
)
]
tracks = [
Track(blockhash=hex(1),
blocknumber=1,
track_id=1,
is_current=True,
is_delete=False,
owner_id=1,
route_id='',
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="the track 1",
download={
"cid": None,
"is_downloadable": False,
"requires_follow": False
}),
Track(blockhash=hex(2),
blocknumber=2,
track_id=2,
is_current=True,
is_delete=False,
owner_id=2,
route_id='',
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="the track 2",
download={
"cid": None,
"is_downloadable": True,
"requires_follow": False
}),
Track(blockhash=hex(3),
blocknumber=3,
track_id=3,
is_current=True,
is_delete=False,
owner_id=1,
route_id='',
track_segments=[],
genre="",
updated_at=now,
created_at=now,
is_unlisted=False,
title="xyz",
download={
"cid": None,
"is_downloadable": True,
"requires_follow": False
})
]
# need users for the lexeme dict to work
users = [
User(blockhash=hex(1),
blocknumber=1,
user_id=1,
is_current=True,
handle="",
wallet="",
updated_at=now,
created_at=now),
User(blockhash=hex(2),
blocknumber=2,
user_id=2,
is_current=True,
handle="",
wallet="",
updated_at=now,
created_at=now),
User(blockhash=hex(3),
blocknumber=3,
user_id=3,
is_current=True,
handle="",
wallet="",
updated_at=now,
created_at=now)
]
with db.scoped_session() as session:
for block in blocks:
session.add(block)
session.flush()
for track in tracks:
session.add(track)
for user in users:
session.add(user)
session.flush()
# Refresh the lexeme matview
session.execute("REFRESH MATERIALIZED VIEW aggregate_track;")
session.execute("REFRESH MATERIALIZED VIEW track_lexeme_dict;")
def test_user_replica_set_indexing_skip_tx(app, mocker):
"""Tests that URSM indexing skips cursed txs without throwing an error and are able to process other tx in block"""
with app.app_context():
db = get_db()
cid_metadata_client = CIDMetadataClient({})
web3 = Web3()
challenge_event_bus = setup_challenge_bus()
update_task = UpdateTask(cid_metadata_client, web3, challenge_event_bus)
class TestUserReplicaSetTransaction:
pass
blessed_user_tx_hash = (
"0x34004dfaf5bb7cf9998eaf387b877d72d198c6508608e309df3f89e57def4db3"
)
blessed_user_tx = TestUserReplicaSetTransaction()
blessed_user_tx.transactionHash = update_task.web3.toBytes(
hexstr=blessed_user_tx_hash
)
cursed_user_tx_hash = (
"0x5fe51d735309d3044ae30055ad29101018a1a399066f6c53ea23800225e3a3be"
)
cursed_user_tx = TestUserReplicaSetTransaction()
cursed_user_tx.transactionHash = update_task.web3.toBytes(
hexstr=cursed_user_tx_hash
)
blessed_cnode_tx_hash = (
"0x42c66d0542383f06e22ef6a235ddba238050d85562bcbd18667c9711c1daee72"
)
blessed_cnode_tx = TestUserReplicaSetTransaction()
blessed_cnode_tx.transactionHash = update_task.web3.toBytes(
hexstr=blessed_cnode_tx_hash
)
cursed_cnode_tx_hash = (
"0xa022761e229302abc2490f8bdc7ec0e642916b0f5cbc2908ccd49498243c1806"
)
cursed_cnode_tx = TestUserReplicaSetTransaction()
cursed_cnode_tx.transactionHash = update_task.web3.toBytes(
hexstr=cursed_cnode_tx_hash
)
test_block_number = 25278765
test_block_timestamp = 1
test_block_hash = update_task.web3.toHex(block_hash)
test_user_replica_set_mgr_txs = [
cursed_user_tx,
blessed_user_tx,
cursed_cnode_tx,
blessed_cnode_tx,
]
test_timestamp = datetime.utcfromtimestamp(test_block_timestamp)
test_wallet = "0x0birbchickemcatlet"
blessed_user_record = User(
blockhash=test_block_hash,
blocknumber=test_block_number,
txhash=blessed_user_tx_hash,
user_id=1,
name="tobey maguire",
is_creator=False,
is_current=True,
updated_at=test_timestamp,
created_at=test_timestamp,
)
cursed_user_record = User(
blockhash=test_block_hash,
blocknumber=test_block_number,
txhash=cursed_user_tx_hash,
user_id=2,
name="birb",
is_current=None,
is_creator=None,
updated_at=test_timestamp,
created_at=None,
)
blessed_content_node_record = URSMContentNode(
blockhash=test_block_hash,
blocknumber=test_block_number,
txhash=blessed_cnode_tx_hash,
is_current=True,
cnode_sp_id=1,
delegate_owner_wallet=test_wallet,
owner_wallet=test_wallet,
created_at=test_timestamp,
)
cursed_content_node_record = URSMContentNode(
blockhash=test_block_hash,
blocknumber=test_block_number,
txhash=cursed_cnode_tx_hash,
is_current=None,
cnode_sp_id=2,
delegate_owner_wallet=test_wallet,
created_at=None,
)
mocker.patch(
"src.tasks.user_replica_set.lookup_user_record",
side_effect=[cursed_user_record, blessed_user_record],
autospec=True,
)
mocker.patch(
"src.tasks.user_replica_set.lookup_ursm_cnode",
side_effect=[cursed_content_node_record, blessed_content_node_record],
autospec=True,
)
mocker.patch(
"src.tasks.user_replica_set.get_endpoint_string_from_sp_ids",
return_value="http://localhost:4001,http://localhost:4002,",
autospec=True,
)
mocker.patch(
"src.tasks.user_replica_set.get_ursm_cnode_endpoint",
return_value="http://localhost:4001,http://localhost:4002,",
autospec=True,
)
mocker.patch(
# because we do not have the real contract set up in the test
# we mock the return value of this fn w events parsed from an imaginary tx
"src.tasks.user_replica_set.get_user_replica_set_mgr_tx",
side_effect=[
[
{
"args": AttrDict(
{
"_userId": cursed_user_record.user_id,
"_primaryId": 1,
"_secondaryIds": [2, 3, 4],
"_signer": "mobey taguire",
}
)
},
], # first tx receipt - update replica set
[], # first tx receipt - update content node
[
{
"args": AttrDict(
{
"_userId": blessed_user_record.user_id,
"_primaryId": 1,
"_secondaryIds": [2, 3, 4],
"_signer": "dirsten kunst",
}
)
},
], # second tx receipt - update replica set
[], # second tx receipt - update content node
[], # third tx receipt - update replica set
[
{
"args": AttrDict(
{
"_cnodeSpId": cursed_content_node_record.cnode_sp_id,
"_cnodeDelegateOwnerWallet": test_wallet,
"_cnodeOwnerWallet": test_wallet,
"_proposer1DelegateOwnerWallet": test_wallet,
"_proposer2DelegateOwnerWallet": test_wallet,
"_proposer3DelegateOwnerWallet": test_wallet,
"_proposerSpIds": [1, 2],
}
)
},
], # third tx receipt - update content node
[], # fourth tx receipt - update replica set
[
{
"args": AttrDict(
{
"_cnodeSpId": blessed_content_node_record.cnode_sp_id,
"_cnodeDelegateOwnerWallet": test_wallet,
"_cnodeOwnerWallet": test_wallet,
"_proposer1DelegateOwnerWallet": test_wallet,
"_proposer2DelegateOwnerWallet": test_wallet,
"_proposer3DelegateOwnerWallet": test_wallet,
"_proposerSpIds": [1, 2],
}
)
},
], # fourth tx receipt - update content node
],
autospec=True,
)
test_ipfs_metadata = {}
test_blacklisted_cids = {}
with db.scoped_session() as session:
try:
current_block = Block(
blockhash=test_block_hash,
parenthash=test_block_hash,
number=test_block_number,
is_current=True,
)
session.add(current_block)
(total_changes, updated_user_ids_set) = user_replica_set_state_update(
update_task,
update_task,
session,
test_user_replica_set_mgr_txs,
test_block_number,
test_block_timestamp,
block_hash,
test_ipfs_metadata,
test_blacklisted_cids,
)
assert len(updated_user_ids_set) == 1
assert list(updated_user_ids_set)[0] == blessed_user_record.user_id
assert total_changes == 2
assert (
session.query(SkippedTransaction)
.filter(
SkippedTransaction.txhash == cursed_cnode_tx_hash,
SkippedTransaction.level == SkippedTransactionLevel.node,
)
.first()
)
assert (
session.query(SkippedTransaction)
.filter(
SkippedTransaction.txhash == cursed_user_tx_hash,
SkippedTransaction.level == SkippedTransactionLevel.node,
)
.first()
)
assert (
session.query(User)
.filter(User.user_id == blessed_user_record.user_id)
.first()
)
assert (
session.query(User)
.filter(User.user_id == cursed_user_record.user_id)
.first()
) == None
assert (
session.query(URSMContentNode)
.filter(
URSMContentNode.cnode_sp_id
== blessed_content_node_record.cnode_sp_id
)
.first()
)
assert (
session.query(URSMContentNode)
.filter(
URSMContentNode.cnode_sp_id
== cursed_content_node_record.cnode_sp_id
)
.first()
) == None
except Exception:
assert False
def test_get_health_verbose(web3_mock, redis_mock, db_mock, get_monitors_mock):
"""Tests that the health check returns verbose db stats"""
get_monitors_mock.return_value = {
'database_connections':
2,
'database_connection_info': [{
'datname': 'audius_discovery',
'state': 'idle',
'query': 'COMMIT',
'wait_event_type': 'Client',
'wait_event': 'ClientRead'
}]
}
# Set up web3 eth
def getBlock(_u1, _u2): # unused
block = MagicMock()
block.number = 2
block.hash = HexBytes(b"\x02")
return block
web3_mock.eth.getBlock = getBlock
# Set up db state
with db_mock.scoped_session() as session:
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash='0x01',
number=1,
parenthash='0x01',
is_current=True,
))
args = {"verbose": True}
health_results, error = get_health(args)
assert error == False
assert health_results["web"]["blocknumber"] == 2
assert health_results["web"]["blockhash"] == "0x02"
assert health_results["db"]["number"] == 1
assert health_results["db"]["blockhash"] == "0x01"
assert health_results["block_difference"] == 1
assert health_results["db_connections"]["database_connections"] == 2
assert health_results["db_connections"]["database_connection_info"] == [{
'datname':
'audius_discovery',
'state':
'idle',
'query':
'COMMIT',
'wait_event_type':
'Client',
'wait_event':
'ClientRead'
}]
assert "maximum_healthy_block_difference" in health_results
assert "version" in health_results
assert "service" in health_results
def test_get_health_verbose(web3_mock, redis_mock, db_mock):
"""Tests that the health check returns verbose db stats"""
# Set up web3 eth
def getBlock(_u1, _u2): # unused
block = MagicMock()
block.number = 2
block.hash = HexBytes(b"\x02")
return block
web3_mock.eth.getBlock = getBlock
# Set up db state
with db_mock.scoped_session() as session:
# Set up mock PG stats
session.execute('CREATE TABLE pg_stat_database (numbackends INTEGER)')
session.execute(
'INSERT INTO pg_stat_database (numbackends) VALUES (1)')
session.execute(
'INSERT INTO pg_stat_database (numbackends) VALUES (1)')
session.execute("""
CREATE TABLE pg_stat_activity (
datname STRING,
query STRING,
state STRING,
wait_event STRING,
wait_event_type STRING
)
""")
session.execute("""
INSERT INTO pg_stat_activity (datname, query, state, wait_event, wait_event_type)
VALUES ("audius_discovery", "COMMIT", "idle", "ClientRead", "Client")
""")
Block.__table__.create(db_mock._engine)
session.add(
Block(
blockhash='0x01',
number=1,
parenthash='0x01',
is_current=True,
))
args = {"verbose": True}
health_results, error = get_health(args)
assert error == False
assert health_results["web"]["blocknumber"] == 2
assert health_results["web"]["blockhash"] == "0x02"
assert health_results["db"]["number"] == 1
assert health_results["db"]["blockhash"] == "0x01"
assert health_results["block_difference"] == 1
assert health_results["db_connections"]["open_connections"] == 2
assert health_results["db_connections"]["connection_info"] == [{
'datname':
'audius_discovery',
'state':
'idle',
'query':
'COMMIT',
'wait_event_type':
'Client',
'wait_event':
'ClientRead'
}]
assert "maximum_healthy_block_difference" in health_results
assert "version" in health_results
assert "service" in health_results
def setup_trending(db, date):
# Test data
# test tracks
# when creating tracks, track_id == index
test_tracks = [
{"genre": "Electronic"},
{"genre": "Pop"},
{"genre": "Electronic"},
# Tracks we don't want to count
{"genre": "Electronic", "is_unlisted": True},
{"genre": "Electronic", "is_delete": True},
]
test_plays = [
# Current Plays
{"item_id": 0},
{"item_id": 0},
{"item_id": 1},
{"item_id": 1},
{"item_id": 2},
{"item_id": 3},
# > 1 wk plays
{"item_id": 2, "created_at": date - timedelta(weeks=2)},
{"item_id": 2, "created_at": date - timedelta(weeks=2)},
{"item_id": 3, "created_at": date - timedelta(weeks=2)},
# We don't want to count these guys (tracks deleted/unlisted)
{"item_id": 3},
{"item_id": 3},
{"item_id": 4},
{"item_id": 4},
]
# pylint: disable=W0621
with db.scoped_session() as session:
# seed tracks + blocks
for i, track_meta in enumerate(test_tracks):
blockhash = hex(i)
block = Block(
blockhash=blockhash,
number=i,
parenthash="0x01",
is_current=True,
)
track = Track(
blockhash=blockhash,
blocknumber=i,
track_id=i,
is_current=track_meta.get("is_current", True),
is_delete=track_meta.get("is_delete", False),
owner_id=300,
route_id="",
track_segments=[],
genre=track_meta.get("genre", ""),
updated_at=track_meta.get("updated_at", date),
created_at=track_meta.get("created_at", date),
is_unlisted=track_meta.get("is_unlisted", False),
)
# add block and then flush before
# adding track, bc track.blocknumber foreign key
# references block
session.add(block)
session.flush()
session.add(track)
# seed plays
aggregate_plays = {}
for i, play_meta in enumerate(test_plays):
item_id = play_meta.get("item_id")
if item_id in aggregate_plays:
aggregate_plays[item_id] += 1
else:
aggregate_plays[item_id] = 1
play = Play(
id=i, play_item_id=item_id, created_at=play_meta.get("created_at", date)
)
session.add(play)
for i, count in aggregate_plays.items():
session.add(AggregatePlays(play_item_id=i, count=count))
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Extracts data from the first block in a .dat file.
Usage: parse_block.py [path to .dat file]
"""
from src.models import Block
from sys import argv
MAX_BLOCK_SIZE = 1024 ** 2 # 1 MB
file_path = argv[1]
with open(file_path) as f:
block = Block.from_data(f.read(MAX_BLOCK_SIZE))
print "Hash:", block.hash()[::-1].encode("hex")
print "Magic number: {:x}".format(block.magic_number)
print "Block size:", block.block_size
print "Version:", block.version
print "Hash of previous block:", block.hash_prev_block[::-1].encode("hex")
print "Hash of merkle root:", block.hash_merkle_root[::-1].encode("hex")
print "Time:", block.time
print "Target: {:064x} ({})".format(block.target, block.bits.encode("hex"))
print "Difficulty:", block.difficulty
print "Pool difficulty:", block.pool_difficulty
print "Nonce:", block.nonce
print "Number of transactions:", block.tx_count
