def fetch_and_parse_sol_rewards_transfer_instruction(
solana_client_manager: SolanaClientManager,
tx_sig: str) -> RewardManagerTransactionInfo:
"""Fetches metadata for rewards transfer transactions and parses data
Fetches the transaction metadata from solana using the tx signature
Checks the metadata for a transfer instruction
Decodes and parses the transfer instruction metadata
Validates the metadata fields
"""
try:
tx_info = solana_client_manager.get_sol_tx_info(tx_sig)
result: TransactionInfoResult = tx_info["result"]
# Create transaction metadata
tx_metadata: RewardManagerTransactionInfo = {
"tx_sig": tx_sig,
"slot": result["slot"],
"timestamp": result["blockTime"],
"transfer_instruction": None,
}
meta = result["meta"]
if meta["err"]:
logger.info(
f"index_rewards_manager.py | Skipping error transaction from chain {tx_info}"
)
return tx_metadata
tx_message = result["transaction"]["message"]
instruction = get_valid_instruction(tx_message, meta)
if instruction is None:
return tx_metadata
transfer_instruction_data = parse_transfer_instruction_data(
instruction["data"])
amount = transfer_instruction_data["amount"]
eth_recipient = transfer_instruction_data["eth_recipient"]
id = transfer_instruction_data["id"]
transfer_instruction = parse_transfer_instruction_id(id)
if transfer_instruction is None:
return tx_metadata
challenge_id, specifier = transfer_instruction
tx_metadata["transfer_instruction"] = {
"amount": amount,
"eth_recipient": eth_recipient,
"challenge_id": challenge_id,
"specifier": specifier,
}
return tx_metadata
except Exception as e:
logger.error(
f"index_rewards_manager.py | Error processing {tx_sig}, {e}",
exc_info=True)
raise e
def fetch_and_cache_latest_program_tx_redis(
solana_client_manager: SolanaClientManager,
redis: Redis,
program: str,
cache_key: str,
):
transactions_history = solana_client_manager.get_signatures_for_address(
program, before=None, limit=1)
transactions_array = transactions_history["result"]
if transactions_array:
# Cache latest transaction from chain
cache_latest_sol_play_program_tx(redis, program, cache_key,
transactions_array[0])
def parse_sol_play_transaction(solana_client_manager: SolanaClientManager,
tx_sig: str):
try:
fetch_start_time = time.time()
tx_info = solana_client_manager.get_sol_tx_info(tx_sig)
fetch_completion_time = time.time()
fetch_time = fetch_completion_time - fetch_start_time
logger.info(
f"index_solana_plays.py | Got transaction: {tx_sig} in {fetch_time}"
)
meta = tx_info["result"]["meta"]
error = meta["err"]
if error:
logger.info(
f"index_solana_plays.py | Skipping error transaction from chain {tx_info}"
)
return None
if is_valid_tx(
tx_info["result"]["transaction"]["message"]["accountKeys"]):
audius_program_index = tx_info["result"]["transaction"]["message"][
"accountKeys"].index(TRACK_LISTEN_PROGRAM)
for instruction in tx_info["result"]["transaction"]["message"][
"instructions"]:
if instruction["programIdIndex"] == audius_program_index:
slot = tx_info["result"]["slot"]
user_id, track_id, source, timestamp = parse_instruction_data(
instruction["data"])
created_at = datetime.utcfromtimestamp(timestamp)
logger.info("index_solana_plays.py | "
f"user_id: {user_id} "
f"track_id: {track_id} "
f"source: {source} "
f"created_at: {created_at} "
f"slot: {slot} "
f"sig: {tx_sig}")
# return the data necessary to create a Play and add to challenge bus
return (user_id, track_id, created_at, source, slot,
tx_sig)
return None
logger.info(
f"index_solana_plays.py | tx={tx_sig} Failed to find SECP_PROGRAM")
return None
except Exception as e:
logger.error(f"index_solana_plays.py | Error processing {tx_sig}, {e}",
exc_info=True)
raise e
def parse_spl_token_transaction(
solana_client_manager: SolanaClientManager,
tx_sig: ConfirmedSignatureForAddressResult,
) -> Tuple[ConfirmedTransaction, List[str], List[str]]:
try:
tx_info = solana_client_manager.get_sol_tx_info(tx_sig["signature"])
result = tx_info["result"]
error = tx_info["result"]["meta"]["err"]
if error:
return (tx_info, [], [])
root_accounts, token_accounts = get_token_balance_change_owners(result)
return (tx_info, list(root_accounts), list(token_accounts))
except Exception as e:
signature = tx_sig["signature"]
logger.error(f"index_spl_token.py | Error processing {signature}, {e}",
exc_info=True)
raise e
def create_celery(test_config=None):
# pylint: disable=W0603
global web3endpoint, web3, abi_values, eth_abi_values, eth_web3
global solana_client_manager
web3endpoint = helpers.get_web3_endpoint(shared_config)
web3 = Web3(HTTPProvider(web3endpoint))
abi_values = helpers.load_abi_values()
# Initialize eth_web3 with MultiProvider
# We use multiprovider to allow for multiple web3 providers and additional resiliency.
# However, we do not use multiprovider in data web3 because of the effect of disparate block status reads.
eth_web3 = Web3(MultiProvider(shared_config["web3"]["eth_provider_url"]))
eth_abi_values = helpers.load_eth_abi_values()
# Initialize Solana web3 provider
solana_client_manager = SolanaClientManager(
shared_config["solana"]["endpoint"])
global registry
global user_factory
global track_factory
global social_feature_factory
global playlist_factory
global user_library_factory
global ipld_blacklist_factory
global user_replica_set_manager
global contract_addresses
# pylint: enable=W0603
(
registry,
user_factory,
track_factory,
social_feature_factory,
playlist_factory,
user_library_factory,
ipld_blacklist_factory,
user_replica_set_manager,
contract_addresses,
) = init_contracts()
return create(test_config, mode="celery")
def parse_user_bank_transaction(
session: Session,
solana_client_manager: SolanaClientManager,
tx_sig,
redis,
challenge_event_bus: ChallengeEventBus,
):
tx_info = solana_client_manager.get_sol_tx_info(tx_sig)
tx_slot = tx_info["result"]["slot"]
timestamp = tx_info["result"]["blockTime"]
parsed_timestamp = datetime.datetime.utcfromtimestamp(timestamp)
logger.info(f"index_user_bank.py | parse_user_bank_transaction |\
{tx_slot}, {tx_sig} | {tx_info} | {parsed_timestamp}")
process_user_bank_tx_details(session, redis, tx_info, tx_sig,
parsed_timestamp, challenge_event_bus)
session.add(
UserBankTransaction(signature=tx_sig,
slot=tx_slot,
created_at=parsed_timestamp))
return (tx_info["result"], tx_sig)
def process_solana_rewards_manager(solana_client_manager: SolanaClientManager,
db: SessionManager, redis: Redis):
"""Fetches the next set of reward manager transactions and updates the DB with Challenge Disbursements"""
if not is_valid_rewards_manager_program:
logger.error(
"index_rewards_manager.py | no valid reward manager program passed"
)
return
if not REWARDS_MANAGER_ACCOUNT:
logger.error(
"index_rewards_manager.py | reward manager account missing")
return
# Get the latests slot available globally before fetching txs to keep track of indexing progress
try:
latest_global_slot = solana_client_manager.get_slot()
except:
logger.error("index_rewards_manager.py | Failed to get slot")
# List of signatures that will be populated as we traverse recent operations
transaction_signatures = get_transaction_signatures(
solana_client_manager,
db,
REWARDS_MANAGER_PROGRAM,
get_latest_reward_disbursment_slot,
get_tx_in_db,
MIN_SLOT,
)
logger.info(f"index_rewards_manager.py | {transaction_signatures}")
last_tx = process_transaction_signatures(solana_client_manager, db, redis,
transaction_signatures)
if last_tx:
redis.set(latest_sol_rewards_manager_slot_key, last_tx["slot"])
elif latest_global_slot is not None:
redis.set(latest_sol_rewards_manager_slot_key, latest_global_slot)
def get_transaction_signatures(
solana_client_manager: SolanaClientManager,
db: SessionManager,
program: str,
get_latest_slot: Callable[[Session], int],
check_tx_exists: Callable[[Session, str], bool],
min_slot=None,
) -> List[List[str]]:
"""Fetches next batch of transaction signature offset from the previous latest processed slot
Fetches the latest processed slot for the rewards manager program
Iterates backwards from the current tx until an intersection is found with the latest processed slot
Returns the next set of transaction signature from the current offset slot to process
"""
# List of signatures that will be populated as we traverse recent operations
transaction_signatures = []
last_tx_signature = None
# Loop exit condition
intersection_found = False
# Query for solana transactions until an intersection is found
with db.scoped_session() as session:
latest_processed_slot = get_latest_slot(session)
while not intersection_found:
transactions_history = solana_client_manager.get_signatures_for_address(
program,
before=last_tx_signature,
limit=FETCH_TX_SIGNATURES_BATCH_SIZE)
transactions_array = transactions_history["result"]
if not transactions_array:
intersection_found = True
logger.info(
f"index_rewards_manager.py | No transactions found before {last_tx_signature}"
)
else:
# Current batch of transactions
transaction_signature_batch = []
for tx_info in transactions_array:
tx_sig = tx_info["signature"]
tx_slot = tx_info["slot"]
logger.info(
f"index_rewards_manager.py | Processing tx={tx_sig} | slot={tx_slot}"
)
if tx_info["slot"] > latest_processed_slot:
transaction_signature_batch.append(tx_sig)
elif tx_info["slot"] <= latest_processed_slot and (
min_slot is None or tx_info["slot"] > min_slot):
# Check the tx signature for any txs in the latest batch,
# and if not present in DB, add to processing
logger.info(
f"index_rewards_manager.py | Latest slot re-traversal\
slot={tx_slot}, sig={tx_sig},\
latest_processed_slot(db)={latest_processed_slot}")
exists = check_tx_exists(session, tx_sig)
if exists:
intersection_found = True
break
# Ensure this transaction is still processed
transaction_signature_batch.append(tx_sig)
# Restart processing at the end of this transaction signature batch
last_tx = transactions_array[-1]
last_tx_signature = last_tx["signature"]
# Append batch of processed signatures
if transaction_signature_batch:
transaction_signatures.append(transaction_signature_batch)
# Ensure processing does not grow unbounded
if len(transaction_signatures) > TX_SIGNATURES_MAX_BATCHES:
# Only take the oldest transaction from the transaction_signatures array
# transaction_signatures is sorted from newest to oldest
transaction_signatures = transaction_signatures[
-TX_SIGNATURES_RESIZE_LENGTH:]
# Reverse batches aggregated so oldest transactions are processed first
transaction_signatures.reverse()
return transaction_signatures
def process_spl_token_tx(solana_client_manager: SolanaClientManager,
db: SessionManager, redis: Redis):
solana_logger = SolanaIndexingLogger("index_spl_token")
solana_logger.start_time("fetch_batches")
try:
base58.b58decode(SPL_TOKEN_PROGRAM)
except ValueError:
logger.error(
f"index_spl_token.py"
f"Invalid Token program ({SPL_TOKEN_PROGRAM}) configured, exiting."
)
return
# Highest currently processed slot in the DB
latest_processed_slot = get_latest_slot(db)
solana_logger.add_log(f"latest used slot: {latest_processed_slot}")
# Utilize the cached tx to offset
cached_offset_tx = fetch_traversed_tx_from_cache(redis,
latest_processed_slot)
# The 'before' value from where we start querying transactions
last_tx_signature = cached_offset_tx
# Loop exit condition
intersection_found = False
# List of signatures that will be populated as we traverse recent operations
transaction_signatures: List[ConfirmedSignatureForAddressResult] = []
# Current batch of transactions
transaction_signature_batch = []
# Current batch
page_count = 0
# Traverse recent records until an intersection is found with latest slot
while not intersection_found:
solana_logger.add_log(
f"Requesting transactions before {last_tx_signature}")
transactions_history = solana_client_manager.get_signatures_for_address(
SPL_TOKEN_PROGRAM,
before=last_tx_signature,
limit=FETCH_TX_SIGNATURES_BATCH_SIZE,
)
solana_logger.add_log(
f"Retrieved transactions before {last_tx_signature}")
transactions_array = transactions_history["result"]
if not transactions_array:
# This is considered an 'intersection' since there are no further transactions to process but
# really represents the end of known history for this ProgramId
intersection_found = True
solana_logger.add_log(
f"No transactions found before {last_tx_signature}")
else:
# handle initial case where no there is no stored latest processed slot and start from current
if latest_processed_slot is None:
logger.info("index_spl_token.py | setting from none")
transaction_signature_batch = transactions_array
intersection_found = True
else:
for tx in transactions_array:
if tx["slot"] > latest_processed_slot:
transaction_signature_batch.append(tx)
elif tx["slot"] <= latest_processed_slot:
intersection_found = True
break
# Restart processing at the end of this transaction signature batch
last_tx = transactions_array[-1]
last_tx_signature = last_tx["signature"]
# Append to recently seen cache
cache_traversed_tx(redis, last_tx)
# Append batch of processed signatures
if transaction_signature_batch:
transaction_signatures.append(transaction_signature_batch)
# Ensure processing does not grow unbounded
if len(transaction_signatures) > TX_SIGNATURES_MAX_BATCHES:
solana_logger.add_log(
f"slicing tx_sigs from {len(transaction_signatures)} entries"
)
transaction_signatures = transaction_signatures[
-TX_SIGNATURES_RESIZE_LENGTH:]
# Reset batch state
transaction_signature_batch = []
solana_logger.add_log(f"intersection_found={intersection_found},\
last_tx_signature={last_tx_signature},\
page_count={page_count}")
page_count = page_count + 1
transaction_signatures.reverse()
logger.info("index_spl_token.py | intersection found")
totals = {"user_ids": 0, "root_accts": 0, "token_accts": 0}
solana_logger.end_time("fetch_batches")
solana_logger.start_time("parse_batches")
for tx_sig_batch in transaction_signatures:
for tx_sig_batch_records in split_list(tx_sig_batch,
TX_SIGNATURES_PROCESSING_SIZE):
user_ids, root_accounts, token_accounts = parse_sol_tx_batch(
db, solana_client_manager, redis, tx_sig_batch_records,
solana_logger)
totals["user_ids"] += len(user_ids)
totals["root_accts"] += len(root_accounts)
totals["token_accts"] += len(token_accounts)
solana_logger.end_time("parse_batches")
solana_logger.add_context("total_user_ids_updated", totals["user_ids"])
solana_logger.add_context("total_root_accts_updated", totals["root_accts"])
solana_logger.add_context("total_token_accts_updated",
totals["token_accts"])
logger.info("index_spl_token.py", extra=solana_logger.get_context())
def process_solana_plays(solana_client_manager: SolanaClientManager,
redis: Redis):
try:
base58.b58decode(TRACK_LISTEN_PROGRAM)
except ValueError:
logger.info(
f"index_solana_plays.py"
f"Invalid TrackListenCount program ({TRACK_LISTEN_PROGRAM}) configured, exiting."
)
return
db = index_solana_plays.db
# Highest currently processed slot in the DB
latest_processed_slot = get_latest_slot(db)
logger.info(
f"index_solana_plays.py | latest used slot: {latest_processed_slot}")
# Utilize the cached tx to offset
cached_offset_tx = fetch_traversed_tx_from_cache(redis,
latest_processed_slot)
# The 'before' value from where we start querying transactions
last_tx_signature = cached_offset_tx
# Loop exit condition
intersection_found = False
# List of signatures that will be populated as we traverse recent operations
transaction_signatures = []
# Current batch of transactions
transaction_signature_batch = []
# Current batch
page_count = 0
# The last transaction processed
last_tx = None
# Get the latests slot available globally before fetching txs to keep track of indexing progress
try:
latest_global_slot = solana_client_manager.get_slot()
except:
logger.error("index_solana_plays.py | Failed to get block height")
# Traverse recent records until an intersection is found with existing Plays table
while not intersection_found:
logger.info(
f"index_solana_plays.py | Requesting transactions before {last_tx_signature}"
)
transactions_history = solana_client_manager.get_signatures_for_address(
TRACK_LISTEN_PROGRAM,
before=last_tx_signature,
limit=FETCH_TX_SIGNATURES_BATCH_SIZE,
)
logger.info(
f"index_solana_plays.py | Retrieved transactions before {last_tx_signature}"
)
transactions_array = transactions_history["result"]
if not transactions_array:
# This is considered an 'intersection' since there are no further transactions to process but
# really represents the end of known history for this ProgramId
intersection_found = True
logger.info(
f"index_solana_plays.py | No transactions found before {last_tx_signature}"
)
else:
with db.scoped_session() as read_session:
for tx in transactions_array:
tx_sig = tx["signature"]
slot = tx["slot"]
if tx["slot"] > latest_processed_slot:
transaction_signature_batch.append(tx_sig)
elif tx["slot"] <= latest_processed_slot:
# Check the tx signature for any txs in the latest batch,
# and if not present in DB, add to processing
logger.info(
f"index_solana_plays.py | Latest slot re-traversal\
slot={slot}, sig={tx_sig},\
latest_processed_slot(db)={latest_processed_slot}")
exists = get_tx_in_db(read_session, tx_sig)
if exists:
# Exit loop and set terminal condition since this tx has been found in DB
# Transactions are returned with most recently committed first, so we can assume
# subsequent transactions in this batch have already been processed
intersection_found = True
break
# Otherwise, ensure this transaction is still processed
transaction_signature_batch.append(tx_sig)
# Restart processing at the end of this transaction signature batch
last_tx = transactions_array[-1]
last_tx_signature = last_tx["signature"]
# Append to recently seen cache
cache_traversed_tx(redis, last_tx)
# Append batch of processed signatures
if transaction_signature_batch:
transaction_signatures.append(transaction_signature_batch)
# Reset batch state
transaction_signature_batch = []
logger.info(
f"index_solana_plays.py | intersection_found={intersection_found},\
last_tx_signature={last_tx_signature},\
page_count={page_count}")
page_count = page_count + 1
transaction_signatures.reverse()
for tx_sig_batch in transaction_signatures:
for tx_sig_batch_records in split_list(tx_sig_batch,
TX_SIGNATURES_PROCESSING_SIZE):
parse_sol_tx_batch(db, solana_client_manager, redis,
tx_sig_batch_records)
try:
if transaction_signatures and transaction_signatures[-1]:
last_tx_sig = transaction_signatures[-1][-1]
tx_info = solana_client_manager.get_sol_tx_info(last_tx_sig)
tx_result: TransactionInfoResult = tx_info["result"]
set_json_cached_key(
redis,
CURRENT_PLAY_INDEXING,
{
"slot": tx_result["slot"],
"timestamp": tx_result["blockTime"]
},
)
except Exception as e:
logger.error(
"index_solana_plays.py | Unable to set redis current play indexing",
exc_info=True,
)
raise e
if last_tx:
redis.set(latest_sol_plays_slot_key, last_tx["slot"])
elif latest_global_slot is not None:
redis.set(latest_sol_plays_slot_key, latest_global_slot)
from unittest import mock
import pytest
from src.solana.solana_client_manager import SolanaClientManager
solana_client_manager = SolanaClientManager(
"https://audius.rpcpool.com,https://api.mainnet-beta.solana.com,https://solana-api.projectserum.com"
)
@mock.patch("solana.rpc.api.Client")
def test_get_client(_):
# test exception raised if no clients
with pytest.raises(Exception):
solana_client_manager.clients = []
solana_client_manager.get_client()
client_mocks = [
mock.Mock(name="first"),
mock.Mock(name="second"),
mock.Mock(name="third"),
]
solana_client_manager.clients = client_mocks
# test that get client returns first one
assert solana_client_manager.get_client() == client_mocks[0]
# test that get random client will sometimes return other clients
num_random_iterations = 10
# very unlikely that 10 calls to get random client all return the first one
returned_other_client = False