def fetch_and_parse_sol_rewards_transfer_instruction(
solana_client_manager: SolanaClientManager,
tx_sig: str) -> RewardManagerTransactionInfo:
"""Fetches metadata for rewards transfer transactions and parses data
Fetches the transaction metadata from solana using the tx signature
Checks the metadata for a transfer instruction
Decodes and parses the transfer instruction metadata
Validates the metadata fields
"""
try:
tx_info = solana_client_manager.get_sol_tx_info(tx_sig)
result: TransactionInfoResult = tx_info["result"]
# Create transaction metadata
tx_metadata: RewardManagerTransactionInfo = {
"tx_sig": tx_sig,
"slot": result["slot"],
"timestamp": result["blockTime"],
"transfer_instruction": None,
}
meta = result["meta"]
if meta["err"]:
logger.info(
f"index_rewards_manager.py | Skipping error transaction from chain {tx_info}"
)
return tx_metadata
tx_message = result["transaction"]["message"]
instruction = get_valid_instruction(tx_message, meta)
if instruction is None:
return tx_metadata
transfer_instruction_data = parse_transfer_instruction_data(
instruction["data"])
amount = transfer_instruction_data["amount"]
eth_recipient = transfer_instruction_data["eth_recipient"]
id = transfer_instruction_data["id"]
transfer_instruction = parse_transfer_instruction_id(id)
if transfer_instruction is None:
return tx_metadata
challenge_id, specifier = transfer_instruction
tx_metadata["transfer_instruction"] = {
"amount": amount,
"eth_recipient": eth_recipient,
"challenge_id": challenge_id,
"specifier": specifier,
}
return tx_metadata
except Exception as e:
logger.error(
f"index_rewards_manager.py | Error processing {tx_sig}, {e}",
exc_info=True)
raise e
def parse_sol_play_transaction(solana_client_manager: SolanaClientManager,
tx_sig: str):
try:
fetch_start_time = time.time()
tx_info = solana_client_manager.get_sol_tx_info(tx_sig)
fetch_completion_time = time.time()
fetch_time = fetch_completion_time - fetch_start_time
logger.info(
f"index_solana_plays.py | Got transaction: {tx_sig} in {fetch_time}"
)
meta = tx_info["result"]["meta"]
error = meta["err"]
if error:
logger.info(
f"index_solana_plays.py | Skipping error transaction from chain {tx_info}"
)
return None
if is_valid_tx(
tx_info["result"]["transaction"]["message"]["accountKeys"]):
audius_program_index = tx_info["result"]["transaction"]["message"][
"accountKeys"].index(TRACK_LISTEN_PROGRAM)
for instruction in tx_info["result"]["transaction"]["message"][
"instructions"]:
if instruction["programIdIndex"] == audius_program_index:
slot = tx_info["result"]["slot"]
user_id, track_id, source, timestamp = parse_instruction_data(
instruction["data"])
created_at = datetime.utcfromtimestamp(timestamp)
logger.info("index_solana_plays.py | "
f"user_id: {user_id} "
f"track_id: {track_id} "
f"source: {source} "
f"created_at: {created_at} "
f"slot: {slot} "
f"sig: {tx_sig}")
# return the data necessary to create a Play and add to challenge bus
return (user_id, track_id, created_at, source, slot,
tx_sig)
return None
logger.info(
f"index_solana_plays.py | tx={tx_sig} Failed to find SECP_PROGRAM")
return None
except Exception as e:
logger.error(f"index_solana_plays.py | Error processing {tx_sig}, {e}",
exc_info=True)
raise e
def parse_spl_token_transaction(
solana_client_manager: SolanaClientManager,
tx_sig: ConfirmedSignatureForAddressResult,
) -> Tuple[ConfirmedTransaction, List[str], List[str]]:
try:
tx_info = solana_client_manager.get_sol_tx_info(tx_sig["signature"])
result = tx_info["result"]
error = tx_info["result"]["meta"]["err"]
if error:
return (tx_info, [], [])
root_accounts, token_accounts = get_token_balance_change_owners(result)
return (tx_info, list(root_accounts), list(token_accounts))
except Exception as e:
signature = tx_sig["signature"]
logger.error(f"index_spl_token.py | Error processing {signature}, {e}",
exc_info=True)
raise e
def parse_user_bank_transaction(
session: Session,
solana_client_manager: SolanaClientManager,
tx_sig,
redis,
challenge_event_bus: ChallengeEventBus,
):
tx_info = solana_client_manager.get_sol_tx_info(tx_sig)
tx_slot = tx_info["result"]["slot"]
timestamp = tx_info["result"]["blockTime"]
parsed_timestamp = datetime.datetime.utcfromtimestamp(timestamp)
logger.info(f"index_user_bank.py | parse_user_bank_transaction |\
{tx_slot}, {tx_sig} | {tx_info} | {parsed_timestamp}")
process_user_bank_tx_details(session, redis, tx_info, tx_sig,
parsed_timestamp, challenge_event_bus)
session.add(
UserBankTransaction(signature=tx_sig,
slot=tx_slot,
created_at=parsed_timestamp))
return (tx_info["result"], tx_sig)
def process_solana_plays(solana_client_manager: SolanaClientManager,
redis: Redis):
try:
base58.b58decode(TRACK_LISTEN_PROGRAM)
except ValueError:
logger.info(
f"index_solana_plays.py"
f"Invalid TrackListenCount program ({TRACK_LISTEN_PROGRAM}) configured, exiting."
)
return
db = index_solana_plays.db
# Highest currently processed slot in the DB
latest_processed_slot = get_latest_slot(db)
logger.info(
f"index_solana_plays.py | latest used slot: {latest_processed_slot}")
# Utilize the cached tx to offset
cached_offset_tx = fetch_traversed_tx_from_cache(redis,
latest_processed_slot)
# The 'before' value from where we start querying transactions
last_tx_signature = cached_offset_tx
# Loop exit condition
intersection_found = False
# List of signatures that will be populated as we traverse recent operations
transaction_signatures = []
# Current batch of transactions
transaction_signature_batch = []
# Current batch
page_count = 0
# The last transaction processed
last_tx = None
# Get the latests slot available globally before fetching txs to keep track of indexing progress
try:
latest_global_slot = solana_client_manager.get_slot()
except:
logger.error("index_solana_plays.py | Failed to get block height")
# Traverse recent records until an intersection is found with existing Plays table
while not intersection_found:
logger.info(
f"index_solana_plays.py | Requesting transactions before {last_tx_signature}"
)
transactions_history = solana_client_manager.get_signatures_for_address(
TRACK_LISTEN_PROGRAM,
before=last_tx_signature,
limit=FETCH_TX_SIGNATURES_BATCH_SIZE,
)
logger.info(
f"index_solana_plays.py | Retrieved transactions before {last_tx_signature}"
)
transactions_array = transactions_history["result"]
if not transactions_array:
# This is considered an 'intersection' since there are no further transactions to process but
# really represents the end of known history for this ProgramId
intersection_found = True
logger.info(
f"index_solana_plays.py | No transactions found before {last_tx_signature}"
)
else:
with db.scoped_session() as read_session:
for tx in transactions_array:
tx_sig = tx["signature"]
slot = tx["slot"]
if tx["slot"] > latest_processed_slot:
transaction_signature_batch.append(tx_sig)
elif tx["slot"] <= latest_processed_slot:
# Check the tx signature for any txs in the latest batch,
# and if not present in DB, add to processing
logger.info(
f"index_solana_plays.py | Latest slot re-traversal\
slot={slot}, sig={tx_sig},\
latest_processed_slot(db)={latest_processed_slot}")
exists = get_tx_in_db(read_session, tx_sig)
if exists:
# Exit loop and set terminal condition since this tx has been found in DB
# Transactions are returned with most recently committed first, so we can assume
# subsequent transactions in this batch have already been processed
intersection_found = True
break
# Otherwise, ensure this transaction is still processed
transaction_signature_batch.append(tx_sig)
# Restart processing at the end of this transaction signature batch
last_tx = transactions_array[-1]
last_tx_signature = last_tx["signature"]
# Append to recently seen cache
cache_traversed_tx(redis, last_tx)
# Append batch of processed signatures
if transaction_signature_batch:
transaction_signatures.append(transaction_signature_batch)
# Reset batch state
transaction_signature_batch = []
logger.info(
f"index_solana_plays.py | intersection_found={intersection_found},\
last_tx_signature={last_tx_signature},\
page_count={page_count}")
page_count = page_count + 1
transaction_signatures.reverse()
for tx_sig_batch in transaction_signatures:
for tx_sig_batch_records in split_list(tx_sig_batch,
TX_SIGNATURES_PROCESSING_SIZE):
parse_sol_tx_batch(db, solana_client_manager, redis,
tx_sig_batch_records)
try:
if transaction_signatures and transaction_signatures[-1]:
last_tx_sig = transaction_signatures[-1][-1]
tx_info = solana_client_manager.get_sol_tx_info(last_tx_sig)
tx_result: TransactionInfoResult = tx_info["result"]
set_json_cached_key(
redis,
CURRENT_PLAY_INDEXING,
{
"slot": tx_result["slot"],
"timestamp": tx_result["blockTime"]
},
)
except Exception as e:
logger.error(
"index_solana_plays.py | Unable to set redis current play indexing",
exc_info=True,
)
raise e
if last_tx:
redis.set(latest_sol_plays_slot_key, last_tx["slot"])
elif latest_global_slot is not None:
redis.set(latest_sol_plays_slot_key, latest_global_slot)