def index_blocks(self, db, blocks_list):
web3 = update_task.web3
redis = update_task.redis
num_blocks = len(blocks_list)
block_order_range = range(len(blocks_list) - 1, -1, -1)
for i in block_order_range:
block = blocks_list[i]
block_index = num_blocks - i
block_number = block.number
block_timestamp = block.timestamp
logger.info(
f"index.py | index_blocks | {self.request.id} | block {block.number} - {block_index}/{num_blocks}"
)
# Handle each block in a distinct transaction
with db.scoped_session() as session:
current_block_query = session.query(Block).filter_by(
is_current=True)
# Without this check we may end up duplicating an insert operation
block_model = Block(
blockhash=web3.toHex(block.hash),
parenthash=web3.toHex(block.parentHash),
number=block.number,
is_current=True,
)
# Update blocks table after
assert (current_block_query.count() == 1
), "Expected single row marked as current"
former_current_block = current_block_query.first()
former_current_block.is_current = False
session.add(block_model)
user_factory_txs = []
track_factory_txs = []
social_feature_factory_txs = []
playlist_factory_txs = []
user_library_factory_txs = []
tx_receipt_dict = fetch_tx_receipts(self, block.transactions)
# Sort transactions by hash
sorted_txs = sorted(block.transactions,
key=lambda entry: entry['hash'])
# Parse tx events in each block
for tx in sorted_txs:
tx_hash = web3.toHex(tx["hash"])
tx_target_contract_address = tx["to"]
tx_receipt = tx_receipt_dict[tx_hash]
# Handle user operations
if tx_target_contract_address == contract_addresses[
"user_factory"]:
logger.info(
f"index.py | index_blocks | UserFactory contract addr: {tx_target_contract_address}"
f" tx from block - {tx}, receipt - {tx_receipt}, adding to user_factory_txs to process in bulk"
)
user_factory_txs.append(tx_receipt)
# Handle track operations
if tx_target_contract_address == contract_addresses[
"track_factory"]:
logger.info(
f"index.py | index_blocks | TrackFactory contract addr: {tx_target_contract_address}"
f" tx from block - {tx}, receipt - {tx_receipt}")
# Track state operations
track_factory_txs.append(tx_receipt)
# Handle social operations
if tx_target_contract_address == contract_addresses[
"social_feature_factory"]:
logger.info(
f"index.py | index_blocks | Social feature contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}")
social_feature_factory_txs.append(tx_receipt)
# Handle repost operations
if tx_target_contract_address == contract_addresses[
"playlist_factory"]:
logger.info(
f"index.py | index_blocks | Playlist contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}")
playlist_factory_txs.append(tx_receipt)
# Handle User Library operations
if tx_target_contract_address == contract_addresses[
"user_library_factory"]:
logger.info(
f"index.py | index_blocks | User Library contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}")
user_library_factory_txs.append(tx_receipt)
# bulk process operations once all tx's for block have been parsed
total_user_changes, user_ids = user_state_update(
self, update_task, session, user_factory_txs, block_number,
block_timestamp)
user_state_changed = total_user_changes > 0
total_track_changes, track_ids = track_state_update(
self, update_task, session, track_factory_txs, block_number,
block_timestamp)
track_state_changed = total_track_changes > 0
social_feature_state_changed = ( # pylint: disable=W0612
social_feature_state_update(self, update_task, session,
social_feature_factory_txs,
block_number, block_timestamp) > 0)
# Playlist state operations processed in bulk
total_playlist_changes, playlist_ids = playlist_state_update(
self, update_task, session, playlist_factory_txs, block_number,
block_timestamp)
playlist_state_changed = total_playlist_changes > 0
user_library_state_changed = user_library_state_update( # pylint: disable=W0612
self, update_task, session, user_library_factory_txs,
block_number, block_timestamp)
track_lexeme_state_changed = (user_state_changed
or track_state_changed)
session.commit()
if user_state_changed:
if user_ids:
remove_cached_user_ids(redis, user_ids)
if track_lexeme_state_changed:
if track_ids:
remove_cached_track_ids(redis, track_ids)
if playlist_state_changed:
if playlist_ids:
remove_cached_playlist_ids(redis, playlist_ids)
# add the block number of the most recently processed block to redis
redis.set(most_recent_indexed_block_redis_key, block.number)
redis.set(most_recent_indexed_block_hash_redis_key, block.hash.hex())
if num_blocks > 0:
logger.warning(
f"index.py | index_blocks | Indexed {num_blocks} blocks")
def index_blocks(self, db, blocks_list):
web3 = update_task.web3
redis = update_task.redis
num_blocks = len(blocks_list)
block_order_range = range(len(blocks_list) - 1, -1, -1)
for i in block_order_range:
if i % 10 == 0 and i != 0:
block_index = num_blocks - i
logger.info(f"index.py | index_blocks | processing block {block_index}/{num_blocks} blocks")
block = blocks_list[i]
block_number = block.number
block_timestamp = block.timestamp
# Handle each block in a distinct transaction
with db.scoped_session() as session:
current_block_query = session.query(Block).filter_by(is_current=True)
block_model = Block(
blockhash=web3.toHex(block.hash),
parenthash=web3.toHex(block.parentHash),
number=block.number,
is_current=True,
)
# Update blocks table after
assert (
current_block_query.count() == 1
), "Expected single row marked as current"
former_current_block = current_block_query.first()
former_current_block.is_current = False
session.add(block_model)
user_factory_txs = []
track_factory_txs = []
social_feature_factory_txs = []
playlist_factory_txs = []
user_library_factory_txs = []
# Sort transactions by hash
sorted_txs = sorted(block.transactions, key=lambda entry: entry['hash'])
# Parse tx events in each block
for tx in sorted_txs:
tx_hash = web3.toHex(tx["hash"])
tx_target_contract_address = tx["to"]
tx_receipt = web3.eth.getTransactionReceipt(tx_hash)
# Handle user operations
if tx_target_contract_address == contract_addresses["user_factory"]:
logger.info(
f"index.py | index_blocks | UserFactory contract addr: {tx_target_contract_address}"
f" tx from block - {tx}, receipt - {tx_receipt}, adding to user_factory_txs to process in bulk"
)
user_factory_txs.append(tx_receipt)
# Handle track operations
if tx_target_contract_address == contract_addresses["track_factory"]:
logger.info(
f"index.py | index_blocks | TrackFactory contract addr: {tx_target_contract_address}"
f" tx from block - {tx}, receipt - {tx_receipt}"
)
# Track state operations
track_factory_txs.append(tx_receipt)
# Handle social operations
if tx_target_contract_address == contract_addresses["social_feature_factory"]:
logger.info(
f"index.py | index_blocks | Social feature contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}"
)
social_feature_factory_txs.append(tx_receipt)
# Handle repost operations
if tx_target_contract_address == contract_addresses["playlist_factory"]:
logger.info(
f"index.py | index_blocks | Playlist contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}"
)
playlist_factory_txs.append(tx_receipt)
# Handle User Library operations
if tx_target_contract_address == contract_addresses["user_library_factory"]:
logger.info(
f"index.py | index_blocks | User Library contract addr: {tx_target_contract_address}"
f"tx from block - {tx}, receipt - {tx_receipt}"
)
user_library_factory_txs.append(tx_receipt)
# bulk process operations once all tx's for block have been parsed
user_state_changed = (
user_state_update(
self, update_task, session, user_factory_txs, block_number, block_timestamp
)
> 0
)
track_state_changed = (
track_state_update(
self, update_task, session, track_factory_txs, block_number, block_timestamp
)
> 0
)
social_feature_state_changed = ( # pylint: disable=W0612
social_feature_state_update(
self, update_task, session, social_feature_factory_txs, block_number, block_timestamp
)
> 0
)
# Playlist state operations processed in bulk
playlist_state_changed = playlist_state_update(
self, update_task, session, playlist_factory_txs, block_number, block_timestamp
)
user_library_state_changed = user_library_state_update( # pylint: disable=W0612
self, update_task, session, user_library_factory_txs, block_number, block_timestamp
)
# keep search materialized view in sync with db
# only refresh track_lexeme_dict when necessary
# social state changes are not factored in since they don't affect track_lexeme_dict
# write out all pending transactions to db before refreshing view
track_lexeme_state_changed = (user_state_changed or track_state_changed)
session.flush()
if user_state_changed:
session.execute("REFRESH MATERIALIZED VIEW user_lexeme_dict")
if track_lexeme_state_changed:
session.execute("REFRESH MATERIALIZED VIEW track_lexeme_dict")
if playlist_state_changed:
session.execute("REFRESH MATERIALIZED VIEW playlist_lexeme_dict")
session.execute("REFRESH MATERIALIZED VIEW album_lexeme_dict")
# add the block number of the most recently processed block to redis
redis.set(most_recent_indexed_block_redis_key, block.number)
if num_blocks > 0:
logger.warning(f"index.py | index_blocks | Indexed {num_blocks} blocks")