def fetch_and_parse_sol_rewards_transfer_instruction( solana_client_manager: SolanaClientManager, tx_sig: str) -> RewardManagerTransactionInfo: """Fetches metadata for rewards transfer transactions and parses data Fetches the transaction metadata from solana using the tx signature Checks the metadata for a transfer instruction Decodes and parses the transfer instruction metadata Validates the metadata fields """ try: tx_info = solana_client_manager.get_sol_tx_info(tx_sig) result: TransactionInfoResult = tx_info["result"] # Create transaction metadata tx_metadata: RewardManagerTransactionInfo = { "tx_sig": tx_sig, "slot": result["slot"], "timestamp": result["blockTime"], "transfer_instruction": None, } meta = result["meta"] if meta["err"]: logger.info( f"index_rewards_manager.py | Skipping error transaction from chain {tx_info}" ) return tx_metadata tx_message = result["transaction"]["message"] instruction = get_valid_instruction(tx_message, meta) if instruction is None: return tx_metadata transfer_instruction_data = parse_transfer_instruction_data( instruction["data"]) amount = transfer_instruction_data["amount"] eth_recipient = transfer_instruction_data["eth_recipient"] id = transfer_instruction_data["id"] transfer_instruction = parse_transfer_instruction_id(id) if transfer_instruction is None: return tx_metadata challenge_id, specifier = transfer_instruction tx_metadata["transfer_instruction"] = { "amount": amount, "eth_recipient": eth_recipient, "challenge_id": challenge_id, "specifier": specifier, } return tx_metadata except Exception as e: logger.error( f"index_rewards_manager.py | Error processing {tx_sig}, {e}", exc_info=True) raise e
def fetch_and_cache_latest_program_tx_redis( solana_client_manager: SolanaClientManager, redis: Redis, program: str, cache_key: str, ): transactions_history = solana_client_manager.get_signatures_for_address( program, before=None, limit=1) transactions_array = transactions_history["result"] if transactions_array: # Cache latest transaction from chain cache_latest_sol_play_program_tx(redis, program, cache_key, transactions_array[0])
def parse_sol_play_transaction(solana_client_manager: SolanaClientManager, tx_sig: str): try: fetch_start_time = time.time() tx_info = solana_client_manager.get_sol_tx_info(tx_sig) fetch_completion_time = time.time() fetch_time = fetch_completion_time - fetch_start_time logger.info( f"index_solana_plays.py | Got transaction: {tx_sig} in {fetch_time}" ) meta = tx_info["result"]["meta"] error = meta["err"] if error: logger.info( f"index_solana_plays.py | Skipping error transaction from chain {tx_info}" ) return None if is_valid_tx( tx_info["result"]["transaction"]["message"]["accountKeys"]): audius_program_index = tx_info["result"]["transaction"]["message"][ "accountKeys"].index(TRACK_LISTEN_PROGRAM) for instruction in tx_info["result"]["transaction"]["message"][ "instructions"]: if instruction["programIdIndex"] == audius_program_index: slot = tx_info["result"]["slot"] user_id, track_id, source, timestamp = parse_instruction_data( instruction["data"]) created_at = datetime.utcfromtimestamp(timestamp) logger.info("index_solana_plays.py | " f"user_id: {user_id} " f"track_id: {track_id} " f"source: {source} " f"created_at: {created_at} " f"slot: {slot} " f"sig: {tx_sig}") # return the data necessary to create a Play and add to challenge bus return (user_id, track_id, created_at, source, slot, tx_sig) return None logger.info( f"index_solana_plays.py | tx={tx_sig} Failed to find SECP_PROGRAM") return None except Exception as e: logger.error(f"index_solana_plays.py | Error processing {tx_sig}, {e}", exc_info=True) raise e
def parse_spl_token_transaction( solana_client_manager: SolanaClientManager, tx_sig: ConfirmedSignatureForAddressResult, ) -> Tuple[ConfirmedTransaction, List[str], List[str]]: try: tx_info = solana_client_manager.get_sol_tx_info(tx_sig["signature"]) result = tx_info["result"] error = tx_info["result"]["meta"]["err"] if error: return (tx_info, [], []) root_accounts, token_accounts = get_token_balance_change_owners(result) return (tx_info, list(root_accounts), list(token_accounts)) except Exception as e: signature = tx_sig["signature"] logger.error(f"index_spl_token.py | Error processing {signature}, {e}", exc_info=True) raise e
def create_celery(test_config=None): # pylint: disable=W0603 global web3endpoint, web3, abi_values, eth_abi_values, eth_web3 global solana_client_manager web3endpoint = helpers.get_web3_endpoint(shared_config) web3 = Web3(HTTPProvider(web3endpoint)) abi_values = helpers.load_abi_values() # Initialize eth_web3 with MultiProvider # We use multiprovider to allow for multiple web3 providers and additional resiliency. # However, we do not use multiprovider in data web3 because of the effect of disparate block status reads. eth_web3 = Web3(MultiProvider(shared_config["web3"]["eth_provider_url"])) eth_abi_values = helpers.load_eth_abi_values() # Initialize Solana web3 provider solana_client_manager = SolanaClientManager( shared_config["solana"]["endpoint"]) global registry global user_factory global track_factory global social_feature_factory global playlist_factory global user_library_factory global ipld_blacklist_factory global user_replica_set_manager global contract_addresses # pylint: enable=W0603 ( registry, user_factory, track_factory, social_feature_factory, playlist_factory, user_library_factory, ipld_blacklist_factory, user_replica_set_manager, contract_addresses, ) = init_contracts() return create(test_config, mode="celery")
def parse_user_bank_transaction( session: Session, solana_client_manager: SolanaClientManager, tx_sig, redis, challenge_event_bus: ChallengeEventBus, ): tx_info = solana_client_manager.get_sol_tx_info(tx_sig) tx_slot = tx_info["result"]["slot"] timestamp = tx_info["result"]["blockTime"] parsed_timestamp = datetime.datetime.utcfromtimestamp(timestamp) logger.info(f"index_user_bank.py | parse_user_bank_transaction |\ {tx_slot}, {tx_sig} | {tx_info} | {parsed_timestamp}") process_user_bank_tx_details(session, redis, tx_info, tx_sig, parsed_timestamp, challenge_event_bus) session.add( UserBankTransaction(signature=tx_sig, slot=tx_slot, created_at=parsed_timestamp)) return (tx_info["result"], tx_sig)
def process_solana_rewards_manager(solana_client_manager: SolanaClientManager, db: SessionManager, redis: Redis): """Fetches the next set of reward manager transactions and updates the DB with Challenge Disbursements""" if not is_valid_rewards_manager_program: logger.error( "index_rewards_manager.py | no valid reward manager program passed" ) return if not REWARDS_MANAGER_ACCOUNT: logger.error( "index_rewards_manager.py | reward manager account missing") return # Get the latests slot available globally before fetching txs to keep track of indexing progress try: latest_global_slot = solana_client_manager.get_slot() except: logger.error("index_rewards_manager.py | Failed to get slot") # List of signatures that will be populated as we traverse recent operations transaction_signatures = get_transaction_signatures( solana_client_manager, db, REWARDS_MANAGER_PROGRAM, get_latest_reward_disbursment_slot, get_tx_in_db, MIN_SLOT, ) logger.info(f"index_rewards_manager.py | {transaction_signatures}") last_tx = process_transaction_signatures(solana_client_manager, db, redis, transaction_signatures) if last_tx: redis.set(latest_sol_rewards_manager_slot_key, last_tx["slot"]) elif latest_global_slot is not None: redis.set(latest_sol_rewards_manager_slot_key, latest_global_slot)
def get_transaction_signatures( solana_client_manager: SolanaClientManager, db: SessionManager, program: str, get_latest_slot: Callable[[Session], int], check_tx_exists: Callable[[Session, str], bool], min_slot=None, ) -> List[List[str]]: """Fetches next batch of transaction signature offset from the previous latest processed slot Fetches the latest processed slot for the rewards manager program Iterates backwards from the current tx until an intersection is found with the latest processed slot Returns the next set of transaction signature from the current offset slot to process """ # List of signatures that will be populated as we traverse recent operations transaction_signatures = [] last_tx_signature = None # Loop exit condition intersection_found = False # Query for solana transactions until an intersection is found with db.scoped_session() as session: latest_processed_slot = get_latest_slot(session) while not intersection_found: transactions_history = solana_client_manager.get_signatures_for_address( program, before=last_tx_signature, limit=FETCH_TX_SIGNATURES_BATCH_SIZE) transactions_array = transactions_history["result"] if not transactions_array: intersection_found = True logger.info( f"index_rewards_manager.py | No transactions found before {last_tx_signature}" ) else: # Current batch of transactions transaction_signature_batch = [] for tx_info in transactions_array: tx_sig = tx_info["signature"] tx_slot = tx_info["slot"] logger.info( f"index_rewards_manager.py | Processing tx={tx_sig} | slot={tx_slot}" ) if tx_info["slot"] > latest_processed_slot: transaction_signature_batch.append(tx_sig) elif tx_info["slot"] <= latest_processed_slot and ( min_slot is None or tx_info["slot"] > min_slot): # Check the tx signature for any txs in the latest batch, # and if not present in DB, add to processing logger.info( f"index_rewards_manager.py | Latest slot re-traversal\ slot={tx_slot}, sig={tx_sig},\ latest_processed_slot(db)={latest_processed_slot}") exists = check_tx_exists(session, tx_sig) if exists: intersection_found = True break # Ensure this transaction is still processed transaction_signature_batch.append(tx_sig) # Restart processing at the end of this transaction signature batch last_tx = transactions_array[-1] last_tx_signature = last_tx["signature"] # Append batch of processed signatures if transaction_signature_batch: transaction_signatures.append(transaction_signature_batch) # Ensure processing does not grow unbounded if len(transaction_signatures) > TX_SIGNATURES_MAX_BATCHES: # Only take the oldest transaction from the transaction_signatures array # transaction_signatures is sorted from newest to oldest transaction_signatures = transaction_signatures[ -TX_SIGNATURES_RESIZE_LENGTH:] # Reverse batches aggregated so oldest transactions are processed first transaction_signatures.reverse() return transaction_signatures
def process_spl_token_tx(solana_client_manager: SolanaClientManager, db: SessionManager, redis: Redis): solana_logger = SolanaIndexingLogger("index_spl_token") solana_logger.start_time("fetch_batches") try: base58.b58decode(SPL_TOKEN_PROGRAM) except ValueError: logger.error( f"index_spl_token.py" f"Invalid Token program ({SPL_TOKEN_PROGRAM}) configured, exiting." ) return # Highest currently processed slot in the DB latest_processed_slot = get_latest_slot(db) solana_logger.add_log(f"latest used slot: {latest_processed_slot}") # Utilize the cached tx to offset cached_offset_tx = fetch_traversed_tx_from_cache(redis, latest_processed_slot) # The 'before' value from where we start querying transactions last_tx_signature = cached_offset_tx # Loop exit condition intersection_found = False # List of signatures that will be populated as we traverse recent operations transaction_signatures: List[ConfirmedSignatureForAddressResult] = [] # Current batch of transactions transaction_signature_batch = [] # Current batch page_count = 0 # Traverse recent records until an intersection is found with latest slot while not intersection_found: solana_logger.add_log( f"Requesting transactions before {last_tx_signature}") transactions_history = solana_client_manager.get_signatures_for_address( SPL_TOKEN_PROGRAM, before=last_tx_signature, limit=FETCH_TX_SIGNATURES_BATCH_SIZE, ) solana_logger.add_log( f"Retrieved transactions before {last_tx_signature}") transactions_array = transactions_history["result"] if not transactions_array: # This is considered an 'intersection' since there are no further transactions to process but # really represents the end of known history for this ProgramId intersection_found = True solana_logger.add_log( f"No transactions found before {last_tx_signature}") else: # handle initial case where no there is no stored latest processed slot and start from current if latest_processed_slot is None: logger.info("index_spl_token.py | setting from none") transaction_signature_batch = transactions_array intersection_found = True else: for tx in transactions_array: if tx["slot"] > latest_processed_slot: transaction_signature_batch.append(tx) elif tx["slot"] <= latest_processed_slot: intersection_found = True break # Restart processing at the end of this transaction signature batch last_tx = transactions_array[-1] last_tx_signature = last_tx["signature"] # Append to recently seen cache cache_traversed_tx(redis, last_tx) # Append batch of processed signatures if transaction_signature_batch: transaction_signatures.append(transaction_signature_batch) # Ensure processing does not grow unbounded if len(transaction_signatures) > TX_SIGNATURES_MAX_BATCHES: solana_logger.add_log( f"slicing tx_sigs from {len(transaction_signatures)} entries" ) transaction_signatures = transaction_signatures[ -TX_SIGNATURES_RESIZE_LENGTH:] # Reset batch state transaction_signature_batch = [] solana_logger.add_log(f"intersection_found={intersection_found},\ last_tx_signature={last_tx_signature},\ page_count={page_count}") page_count = page_count + 1 transaction_signatures.reverse() logger.info("index_spl_token.py | intersection found") totals = {"user_ids": 0, "root_accts": 0, "token_accts": 0} solana_logger.end_time("fetch_batches") solana_logger.start_time("parse_batches") for tx_sig_batch in transaction_signatures: for tx_sig_batch_records in split_list(tx_sig_batch, TX_SIGNATURES_PROCESSING_SIZE): user_ids, root_accounts, token_accounts = parse_sol_tx_batch( db, solana_client_manager, redis, tx_sig_batch_records, solana_logger) totals["user_ids"] += len(user_ids) totals["root_accts"] += len(root_accounts) totals["token_accts"] += len(token_accounts) solana_logger.end_time("parse_batches") solana_logger.add_context("total_user_ids_updated", totals["user_ids"]) solana_logger.add_context("total_root_accts_updated", totals["root_accts"]) solana_logger.add_context("total_token_accts_updated", totals["token_accts"]) logger.info("index_spl_token.py", extra=solana_logger.get_context())
def process_solana_plays(solana_client_manager: SolanaClientManager, redis: Redis): try: base58.b58decode(TRACK_LISTEN_PROGRAM) except ValueError: logger.info( f"index_solana_plays.py" f"Invalid TrackListenCount program ({TRACK_LISTEN_PROGRAM}) configured, exiting." ) return db = index_solana_plays.db # Highest currently processed slot in the DB latest_processed_slot = get_latest_slot(db) logger.info( f"index_solana_plays.py | latest used slot: {latest_processed_slot}") # Utilize the cached tx to offset cached_offset_tx = fetch_traversed_tx_from_cache(redis, latest_processed_slot) # The 'before' value from where we start querying transactions last_tx_signature = cached_offset_tx # Loop exit condition intersection_found = False # List of signatures that will be populated as we traverse recent operations transaction_signatures = [] # Current batch of transactions transaction_signature_batch = [] # Current batch page_count = 0 # The last transaction processed last_tx = None # Get the latests slot available globally before fetching txs to keep track of indexing progress try: latest_global_slot = solana_client_manager.get_slot() except: logger.error("index_solana_plays.py | Failed to get block height") # Traverse recent records until an intersection is found with existing Plays table while not intersection_found: logger.info( f"index_solana_plays.py | Requesting transactions before {last_tx_signature}" ) transactions_history = solana_client_manager.get_signatures_for_address( TRACK_LISTEN_PROGRAM, before=last_tx_signature, limit=FETCH_TX_SIGNATURES_BATCH_SIZE, ) logger.info( f"index_solana_plays.py | Retrieved transactions before {last_tx_signature}" ) transactions_array = transactions_history["result"] if not transactions_array: # This is considered an 'intersection' since there are no further transactions to process but # really represents the end of known history for this ProgramId intersection_found = True logger.info( f"index_solana_plays.py | No transactions found before {last_tx_signature}" ) else: with db.scoped_session() as read_session: for tx in transactions_array: tx_sig = tx["signature"] slot = tx["slot"] if tx["slot"] > latest_processed_slot: transaction_signature_batch.append(tx_sig) elif tx["slot"] <= latest_processed_slot: # Check the tx signature for any txs in the latest batch, # and if not present in DB, add to processing logger.info( f"index_solana_plays.py | Latest slot re-traversal\ slot={slot}, sig={tx_sig},\ latest_processed_slot(db)={latest_processed_slot}") exists = get_tx_in_db(read_session, tx_sig) if exists: # Exit loop and set terminal condition since this tx has been found in DB # Transactions are returned with most recently committed first, so we can assume # subsequent transactions in this batch have already been processed intersection_found = True break # Otherwise, ensure this transaction is still processed transaction_signature_batch.append(tx_sig) # Restart processing at the end of this transaction signature batch last_tx = transactions_array[-1] last_tx_signature = last_tx["signature"] # Append to recently seen cache cache_traversed_tx(redis, last_tx) # Append batch of processed signatures if transaction_signature_batch: transaction_signatures.append(transaction_signature_batch) # Reset batch state transaction_signature_batch = [] logger.info( f"index_solana_plays.py | intersection_found={intersection_found},\ last_tx_signature={last_tx_signature},\ page_count={page_count}") page_count = page_count + 1 transaction_signatures.reverse() for tx_sig_batch in transaction_signatures: for tx_sig_batch_records in split_list(tx_sig_batch, TX_SIGNATURES_PROCESSING_SIZE): parse_sol_tx_batch(db, solana_client_manager, redis, tx_sig_batch_records) try: if transaction_signatures and transaction_signatures[-1]: last_tx_sig = transaction_signatures[-1][-1] tx_info = solana_client_manager.get_sol_tx_info(last_tx_sig) tx_result: TransactionInfoResult = tx_info["result"] set_json_cached_key( redis, CURRENT_PLAY_INDEXING, { "slot": tx_result["slot"], "timestamp": tx_result["blockTime"] }, ) except Exception as e: logger.error( "index_solana_plays.py | Unable to set redis current play indexing", exc_info=True, ) raise e if last_tx: redis.set(latest_sol_plays_slot_key, last_tx["slot"]) elif latest_global_slot is not None: redis.set(latest_sol_plays_slot_key, latest_global_slot)
from unittest import mock import pytest from src.solana.solana_client_manager import SolanaClientManager solana_client_manager = SolanaClientManager( "https://audius.rpcpool.com,https://api.mainnet-beta.solana.com,https://solana-api.projectserum.com" ) @mock.patch("solana.rpc.api.Client") def test_get_client(_): # test exception raised if no clients with pytest.raises(Exception): solana_client_manager.clients = [] solana_client_manager.get_client() client_mocks = [ mock.Mock(name="first"), mock.Mock(name="second"), mock.Mock(name="third"), ] solana_client_manager.clients = client_mocks # test that get client returns first one assert solana_client_manager.get_client() == client_mocks[0] # test that get random client will sometimes return other clients num_random_iterations = 10 # very unlikely that 10 calls to get random client all return the first one returned_other_client = False