async def test_transactions_get_queued_in_queue_is_weird(self, *, monitor, manager):
"""This test reproduces a bug where if there was a transaction with too low a gas price queued,
the monitor sanity check would force any further queued transactions with a good gas price and
cause the previous transaction to the error state due to the nonce being problematic.
This tests that even if the queue gets in a weird state like this it still gets resolved correctly
"""
gas_price = 50000000000
safe_low_gas_price = 40000000000
too_low_gas_price = 30000000000
assert(gas_price != DEFAULT_GASPRICE)
assert(gas_price > safe_low_gas_price)
assert(safe_low_gas_price > too_low_gas_price)
await self.redis.set("gas_station_standard_gas_price", hex(gas_price))
await self.redis.set("gas_station_safelow_gas_price", hex(safe_low_gas_price))
for nonce, gasprice, value in [(0, too_low_gas_price, 0x1000000000000),
(1, safe_low_gas_price, 0x2000000000000),
(2, gas_price, 0x3000000000000)]:
tx = create_transaction(nonce=0x100000 + nonce, gasprice=gasprice, startgas=DEFAULT_STARTGAS,
to=TEST_ADDRESS, value=value, network_id=0x42)
await self.sign_and_send_tx(FAUCET_PRIVATE_KEY, encode_transaction(tx))
await asyncio.sleep(0.1)
async with self.pool.acquire() as con:
txs = await con.fetch("SELECT * FROM transactions")
for tx in txs:
self.assertEqual(tx['status'], 'queued')
# purposefully cause some problems
async with self.pool.acquire() as con:
txs = await con.fetch("UPDATE transactions SET status = 'unconfirmed' WHERE nonce > $1", 0x100000)
from toshieth.tasks import manager_dispatcher
manager_dispatcher.process_transaction_queue(FAUCET_ADDRESS)
await asyncio.sleep(0.1)
async with self.pool.acquire() as con:
tx = await con.fetchrow("SELECT * FROM transactions WHERE nonce = $1", 0x100000)
self.assertEqual(tx['status'], 'queued')
# fix up gas prices
await self.redis.set("gas_station_standard_gas_price", hex(safe_low_gas_price))
await self.redis.set("gas_station_safelow_gas_price", hex(too_low_gas_price))
manager_dispatcher.process_transaction_queue(FAUCET_ADDRESS)
await self.wait_on_tx_confirmation(tx['hash'])
async with self.pool.acquire() as con:
tx = await con.fetchrow("SELECT * FROM transactions WHERE nonce = $1", 0x100000)
self.assertEqual(tx['status'], 'confirmed')
async def send_transaction(self, *, tx, signature=None):
try:
tx = decode_transaction(tx)
except:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Invalid Transaction'
})
if is_transaction_signed(tx):
tx_sig = data_encoder(signature_from_transaction(tx))
if signature:
if tx_sig != signature:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: Signature in payload and signature of transaction do not match'
})
else:
signature = tx_sig
else:
if signature is None:
raise JsonRPCInvalidParamsError(data={
'id': 'missing_signature',
'message': 'Missing Signature'
})
if not validate_signature(signature):
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: {}'.format('Invalid length' if len(
signature) != 132 else 'Invalid hex value')
})
try:
sig = data_decoder(signature)
except Exception:
log.exception(
"Unexpected error decoding valid signature: {}".format(
signature))
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature'
})
add_signature_to_transaction(tx, sig)
# validate network id, if it's not for "all networks"
if tx.network_id is not None and self.network_id != tx.network_id:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_network_id',
'message': 'Invalid Network ID'
})
from_address = data_encoder(tx.sender)
to_address = data_encoder(tx.to)
# prevent spamming of transactions with the same nonce from the same sender
async with RedisLock("{}:{}".format(from_address, tx.nonce),
raise_when_locked=partial(
JsonRPCInvalidParamsError,
data={
'id': 'invalid_nonce',
'message': 'Nonce already used'
}),
ex=5):
# check for transaction overwriting
async with self.db:
existing = await self.db.fetchrow(
"SELECT * FROM transactions WHERE "
"from_address = $1 AND nonce = $2 AND "
"(status != 'error' or status is NULL)", from_address,
tx.nonce)
# disallow transaction overwriting when the gas is lower or the transaction is confirmed
if existing and (parse_int(existing['gas_price']) >= tx.gasprice
or existing['status'] == 'confirmed'):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_nonce',
'message': 'Nonce already used'
})
# make sure the account has enough funds for the transaction
network_balance, balance, _, _ = await self.get_balances(
from_address)
if existing:
balance += parse_int(existing['value']) + parse_int(
existing['gas']) * parse_int(existing['gas_price'])
if balance < (tx.value + (tx.startgas * tx.gasprice)):
raise JsonRPCInsufficientFundsError(
data={
'id': 'insufficient_funds',
'message': 'Insufficient Funds'
})
# validate the nonce (only necessary if tx doesn't already exist)
if not existing:
c_nonce = await self.get_transaction_count(from_address)
if tx.nonce < c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too low'
})
if tx.nonce > c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too high'
})
if tx.intrinsic_gas_used > tx.startgas:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_transaction',
'message':
'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'
.format(tx.intrinsic_gas_used, tx.startgas)
})
# now this tx fits enough of the criteria to allow it
# onto the transaction queue
tx_hash = calculate_transaction_hash(tx)
if existing:
log.info(
"Setting tx '{}' to error due to forced overwrite".format(
existing['hash']))
manager_dispatcher.update_transaction(
existing['transaction_id'], 'error')
data = data_encoder(tx.data)
if data and \
((data.startswith("0xa9059cbb") and len(data) == 138) or \
(data.startswith("0x23b872dd") and len(data) == 202)):
# check if the token is a known erc20 token
async with self.db:
erc20_token = await self.db.fetchrow(
"SELECT * FROM tokens WHERE contract_address = $1",
to_address)
else:
erc20_token = False
# add tx to database
async with self.db:
db_tx = await self.db.fetchrow(
"INSERT INTO transactions "
"(hash, from_address, to_address, nonce, "
"value, gas, gas_price, "
"data, v, r, s, "
"sender_toshi_id) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) "
"RETURNING transaction_id", tx_hash, from_address,
to_address, tx.nonce, hex(tx.value), hex(tx.startgas),
hex(tx.gasprice), data_encoder(tx.data), hex(tx.v),
hex(tx.r), hex(tx.s), self.user_toshi_id)
if erc20_token:
token_value = int(data[-64:], 16)
if data.startswith("0x23b872dd"):
erc20_from_address = "0x" + data[34:74]
erc20_to_address = "0x" + data[98:138]
else:
erc20_from_address = from_address
erc20_to_address = "0x" + data[34:74]
await self.db.execute(
"INSERT INTO token_transactions "
"(transaction_id, transaction_log_index, contract_address, from_address, to_address, value) "
"VALUES ($1, $2, $3, $4, $5, $6)",
db_tx['transaction_id'], 0,
erc20_token['contract_address'], erc20_from_address,
erc20_to_address, hex(token_value))
await self.db.commit()
# trigger processing the transaction queue
manager_dispatcher.process_transaction_queue(from_address)
# analytics
# use notification registrations to try find toshi ids for users
if self.user_toshi_id:
sender_toshi_id = self.user_toshi_id
else:
async with self.db:
sender_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", from_address)
async with self.db:
receiver_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", to_address)
self.track(sender_toshi_id, "Sent transaction")
# it doesn't make sense to add user agent here as we
# don't know the receiver's user agent
self.track(receiver_toshi_id,
"Received transaction",
add_user_agent=False)
return tx_hash
async def _process_transaction_queue(self, ethereum_address):
log.debug("processing tx queue for {}".format(ethereum_address))
# check for un-scheduled transactions
async with self.db:
# get the last block number to use in ethereum calls
# to avoid race conditions in transactions being confirmed
# on the network before the block monitor sees and updates them in the database
last_blocknumber = (
await
self.db.fetchval("SELECT blocknumber FROM last_blocknumber"))
transactions_out = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'new' OR status = 'queued') "
"AND r IS NOT NULL "
# order by nonce reversed so that .pop() can
# be used in the loop below
"ORDER BY nonce DESC",
ethereum_address)
# any time the state of a transaction is changed we need to make
# sure those changes cascade down to the receiving address as well
# this keeps a list of all the receiving addresses that need to be
# checked after the current address's queue has been processed
addresses_to_check = set()
if transactions_out:
# TODO: make sure the block number isn't too far apart from the current
# if this is the case then we should just come back later!
# get the current network balance for this address
balance = await self.eth.eth_getBalance(ethereum_address,
block=last_blocknumber
or "latest")
# get the unconfirmed_txs
async with self.db:
unconfirmed_txs = await self.db.fetch(
"SELECT nonce, value, gas, gas_price FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'unconfirmed' "
"OR (status = 'confirmed' AND blocknumber > $2)) "
"ORDER BY nonce", ethereum_address, last_blocknumber or 0)
network_nonce = await self.eth.eth_getTransactionCount(
ethereum_address, block=last_blocknumber or "latest")
if unconfirmed_txs:
nonce = unconfirmed_txs[-1]['nonce'] + 1
balance -= sum(
parse_int(tx['value']) +
(parse_int(tx['gas']) * parse_int(tx['gas_price']))
for tx in unconfirmed_txs)
else:
# use the nonce from the network
nonce = network_nonce
# marker for whether a previous transaction had an error (signaling
# that all the following should also be an error
previous_error = False
# for each one, check if we can schedule them yet
while transactions_out:
transaction = transactions_out.pop()
# if there was a previous error in the queue, abort!
if previous_error:
log.info("Setting tx '{}' to error due to previous error".
format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# make sure the nonce is still valid
if nonce != transaction[
'nonce'] and network_nonce != transaction['nonce']:
# check if this is an overwrite
if transaction['status'] == 'new':
async with self.db:
old_tx = await self.db.fetchrow(
"SELECT * FROM transactions where from_address = $1 AND nonce = $2 AND hash != $3",
ethereum_address, transaction['nonce'],
transaction['hash'])
if old_tx:
if old_tx['status'] == 'error':
# expected state for overwrites
pass
elif old_tx['status'] == 'unconfirmed' or old_tx[
'status'] == 'confirmed':
previous_error = True
log.info((
"Setting tx '{}' to error due to another unconfirmed transaction"
"with nonce ({}) already existing in the system"
).format(transaction['hash'],
transaction['nonce']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(
transaction['to_address'])
continue
else:
# two transactions with the same nonce on the queue
# lets pick the one with the highest gas price and error the other
if transaction['nonce'] > old_tx['nonce']:
# lets use this one!
log.info((
"Setting tx '{}' to error due to another unconfirmed transaction"
"with nonce ({}) already existing in the system"
).format(old_tx['hash'],
transaction['nonce']))
await self.update_transaction(
old_tx['transaction_id'], 'error')
addresses_to_check.add(
old_tx['to_address'])
# make sure the other transaction is pulled out of the queue
try:
idx = next(i for i, e in enumerate(
transactions_out)
if e['transaction_id'] ==
old_tx['transaction_id'])
del transactions_out[idx]
except:
# old_tx not in the transactions_out list
pass
else:
# we'll use the other one
log.info((
"Setting tx '{}' to error due to another unconfirmed transaction"
"with nonce ({}) already existing in the system"
).format(old_tx['hash'],
transaction['nonce']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(
transaction['to_address'])
addresses_to_check.add(
transaction['from_address'])
# this case is actually pretty weird, so emptying the
# transactions_out so we restart the queue check
# completely
transactions_out = []
continue
else:
# well this is awkward! may as well let things go on in this case because
# it means a transaction in the nonce sequence is missing
pass
elif transaction['status'] == 'queued':
# then this and all the following transactions are now invalid
previous_error = True
log.info(
"Setting tx '{}' to error due to the nonce ({}) not matching the network ({})"
.format(transaction['hash'], transaction['nonce'],
nonce))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
else:
# this is a really weird state
# it's not clear what should be done here
log.error(
"Found unconfirmed transaction with out of order nonce for address: {}"
.format(ethereum_address))
return
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
cost = value + (gas * gas_price)
# check if the current balance is high enough to send to the network
if balance >= cost:
# check if gas price is high enough that it makes sense to send the transaction
safe_gas_price = parse_int(
await self.redis.get('gas_station_safelow_gas_price'))
if safe_gas_price and safe_gas_price > gas_price:
log.debug(
"Not queuing tx '{}' as current gas price would not support it"
.format(transaction['hash']))
# retry this address in a minute
manager_dispatcher.process_transaction_queue(
ethereum_address).delay(60)
# abort the rest of the processing after sending PNs for any "new" transactions
while transaction:
if transaction['status'] == 'new':
await self.update_transaction(
transaction['transaction_id'], 'queued')
transaction = transactions_out.pop(
) if transactions_out else None
break
# if so, send the transaction
# create the transaction
data = data_decoder(
transaction['data']) if transaction['data'] else b''
tx = create_transaction(nonce=transaction['nonce'],
value=value,
gasprice=gas_price,
startgas=gas,
to=transaction['to_address'],
data=data,
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
# make sure the signature was valid
if data_encoder(tx.sender) != ethereum_address:
# signature is invalid for the user
log.error(
"ERROR signature invalid for sender of tx: {}".
format(transaction['hash']))
log.error("queue: {}, db: {}, tx: {}".format(
ethereum_address, transaction['from_address'],
data_encoder(tx.sender)))
previous_error = True
addresses_to_check.add(transaction['to_address'])
await self.update_transaction(
transaction['transaction_id'], 'error')
continue
# send the transaction
try:
tx_encoded = encode_transaction(tx)
await self.eth.eth_sendRawTransaction(tx_encoded)
await self.update_transaction(
transaction['transaction_id'], 'unconfirmed')
except JsonRPCError as e:
# if something goes wrong with sending the transaction
# simply abort for now.
# TODO: depending on error, just break and queue to retry later
log.error(
"ERROR sending queued transaction: {}".format(
e.format()))
if e.message and (e.message.startswith(
"Transaction nonce is too low"
) or e.message.startswith(
"Transaction with the same hash was already imported"
)):
existing_tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
if existing_tx:
if existing_tx['blockNumber']:
await self.update_transaction(
transaction['transaction_id'],
'confirmed')
else:
await self.update_transaction(
transaction['transaction_id'],
'unconfirmed')
continue
previous_error = True
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# adjust the balance values for checking the other transactions
balance -= cost
if nonce == transaction['nonce']:
nonce += 1
continue
else:
# make sure the pending_balance would support this transaction
# otherwise there's no way this transaction will be able to
# be send, so trigger a failure on all the remaining transactions
async with self.db:
transactions_in = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE to_address = $1 "
"AND ("
"(status = 'new' OR status = 'queued' OR status = 'unconfirmed') "
"OR (status = 'confirmed' AND blocknumber > $2))",
ethereum_address, last_blocknumber or 0)
# TODO: test if loops in the queue chain are problematic
pending_received = sum(
(parse_int(p['value']) or 0) for p in transactions_in)
if balance + pending_received < cost:
previous_error = True
log.info(
"Setting tx '{}' to error due to insufficient pending balance"
.format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
else:
if any(t['blocknumber'] is not None
and t['blocknumber'] > last_blocknumber
for t in transactions_in):
addresses_to_check.add(ethereum_address)
# there's no reason to continue on here since all the
# following transaction in the queue cannot be processed
# until this one is
# but we still need to send PNs for any "new" transactions
while transaction:
if transaction['status'] == 'new':
await self.update_transaction(
transaction['transaction_id'], 'queued')
transaction = transactions_out.pop(
) if transactions_out else None
break
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
manager_dispatcher.process_transaction_queue(address)
if transactions_out:
manager_dispatcher.process_transaction_queue(ethereum_address)
async def sanity_check(self, frequency):
async with self.db:
rows = await self.db.fetch(
"SELECT DISTINCT from_address FROM transactions WHERE (status = 'unconfirmed' OR status = 'queued' OR status = 'new') "
"AND v IS NOT NULL AND created < (now() AT TIME ZONE 'utc') - interval '3 minutes'"
)
rows2 = await self.db.fetch(
"WITH t1 AS (SELECT DISTINCT from_address FROM transactions WHERE (status = 'new' OR status = 'queued') AND v IS NOT NULL), "
"t2 AS (SELECT from_address, COUNT(*) FROM transactions WHERE (status = 'unconfirmed' AND v IS NOT NULL) GROUP BY from_address) "
"SELECT t1.from_address FROM t1 LEFT JOIN t2 ON t1.from_address = t2.from_address WHERE t2.count IS NULL;"
)
if rows or rows2:
log.debug(
"sanity check found {} addresses with potential problematic transactions"
.format(len(rows) + len(rows2)))
rows = set([row['from_address'] for row in rows
]).union(set([row['from_address'] for row in rows2]))
addresses_to_check = set()
old_and_unconfirmed = []
for ethereum_address in rows:
# check on queued transactions
async with self.db:
queued_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'new' OR status = 'queued') AND v IS NOT NULL",
ethereum_address)
if queued_transactions:
# make sure there are pending incoming transactions
async with self.db:
incoming_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE to_address = $1 "
"AND (status = 'unconfirmed' OR status = 'queued' OR status = 'new')",
ethereum_address)
if not incoming_transactions:
log.error(
"ERROR: {} has transactions in it's queue, but no unconfirmed transactions!"
.format(ethereum_address))
# trigger queue processing as last resort
addresses_to_check.add(ethereum_address)
else:
# check health of the incoming transaction
for transaction in incoming_transactions:
if transaction['v'] is None:
try:
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
except:
log.exception(
"Error getting transaction {} in sanity check",
transaction['hash'])
continue
if tx is None:
log.warning(
"external transaction (id: {}) no longer found on nodes"
.format(transaction['transaction_id']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(ethereum_address)
elif tx['blockNumber'] is not None:
log.warning(
"external transaction (id: {}) confirmed on node, but wasn't confirmed in db"
.format(transaction['transaction_id']))
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(ethereum_address)
# no need to continue with dealing with unconfirmed transactions if there are queued ones
continue
async with self.db:
unconfirmed_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND status = 'unconfirmed' AND v IS NOT NULL",
ethereum_address)
if unconfirmed_transactions:
for transaction in unconfirmed_transactions:
# check on unconfirmed transactions first
if transaction['status'] == 'unconfirmed':
# we neehed to check the true status of unconfirmed transactions
# as the block monitor may be inbetween calls and not have seen
# this transaction to mark it as confirmed.
try:
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
except:
log.exception(
"Error getting transaction {} in sanity check",
transaction['hash'])
continue
# sanity check to make sure the tx still exists
if tx is None:
# if not, try resubmit
# NOTE: it may just be an issue with load balanced nodes not seeing all pending transactions
# so we don't want to adjust the status of the transaction at all at this stage
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
data = data_decoder(
transaction['data']
) if transaction['data'] else b''
tx = create_transaction(
nonce=transaction['nonce'],
value=value,
gasprice=gas_price,
startgas=gas,
to=transaction['to_address'],
data=data,
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
if calculate_transaction_hash(
tx) != transaction['hash']:
log.warning(
"error resubmitting transaction {}: regenerating tx resulted in a different hash"
.format(transaction['hash']))
else:
tx_encoded = encode_transaction(tx)
try:
await self.eth.eth_sendRawTransaction(
tx_encoded)
addresses_to_check.add(
transaction['from_address'])
except Exception as e:
# note: usually not critical, don't panic
log.warning(
"error resubmitting transaction {}: {}"
.format(transaction['hash'], str(e)))
elif tx['blockNumber'] is not None:
# confirmed! update the status
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(transaction['from_address'])
addresses_to_check.add(transaction['to_address'])
else:
old_and_unconfirmed.append(transaction['hash'])
if len(old_and_unconfirmed):
log.warning(
"WARNING: {} transactions are old and unconfirmed!".format(
len(old_and_unconfirmed)))
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
manager_dispatcher.process_transaction_queue(address)
if frequency:
manager_dispatcher.sanity_check(frequency).delay(frequency)
async def update_transaction(self,
transaction_id,
status,
retry_start_time=0):
async with self.db:
tx = await self.db.fetchrow(
"SELECT * FROM transactions WHERE transaction_id = $1",
transaction_id)
if tx is None or tx['status'] == status:
return
token_txs = await self.db.fetch(
"SELECT tok.symbol, tok.name, tok.decimals, tx.contract_address, tx.value, tx.from_address, tx.to_address, tx.transaction_log_index, tx.status "
"FROM token_transactions tx "
"JOIN tokens tok "
"ON tok.contract_address = tx.contract_address "
"WHERE tx.transaction_id = $1", transaction_id)
# check if we're trying to update the state of a tx that is already confirmed, we have an issue
if tx['status'] == 'confirmed':
log.warning(
"Trying to update status of tx {} to {}, but tx is already confirmed"
.format(tx['hash'], status))
return
# only log if the transaction is internal
if tx['v'] is not None:
log.info(
"Updating status of tx {} to {} (previously: {})".format(
tx['hash'], status, tx['status']))
if status == 'confirmed':
try:
bulk = self.eth.bulk()
transaction = bulk.eth_getTransactionByHash(tx['hash'])
tx_receipt = bulk.eth_getTransactionReceipt(tx['hash'])
await bulk.execute()
transaction = transaction.result()
tx_receipt = tx_receipt.result()
except:
log.exception("Error getting transaction: {}".format(
tx['hash']))
transaction = None
tx_receipt = None
if transaction and 'blockNumber' in transaction and transaction[
'blockNumber'] is not None:
if retry_start_time > 0:
log.info(
"successfully confirmed tx {} after {} seconds".format(
tx['hash'], round(time.time() - retry_start_time,
2)))
token_tx_updates = []
updated_token_txs = []
for token_tx in token_txs:
from_address = token_tx['from_address']
to_address = token_tx['to_address']
# check transaction receipt to make sure the transfer was successful
has_transfer_event = False
token_tx_status = 'confirmed'
if tx_receipt[
'logs'] is not None: # should always be [], but checking just incase
for _log in tx_receipt['logs']:
if len(_log['topics']) > 0 and _log['topics'][
0] == TRANSFER_TOPIC:
if len(_log['topics']) == 3 and len(_log['data']) == 66 and \
decode_single_address(_log['topics'][1]) == from_address and \
decode_single_address(_log['topics'][2]) == to_address:
has_transfer_event = True
break
elif len(_log['topics']) == 1 and len(
_log['data']) == 194:
erc20_from_address, erc20_to_address, erc20_value = decode_abi(
['address', 'address', 'uint256'],
data_decoder(_log['data']))
if erc20_from_address == from_address and \
erc20_to_address == to_address:
has_transfer_event = True
break
elif _log['address'] == WETH_CONTRACT_ADDRESS:
if _log['topics'][
0] == DEPOSIT_TOPIC and decode_single_address(
_log['topics'][1]) == to_address:
has_transfer_event = True
break
elif _log['topics'][
0] == WITHDRAWAL_TOPIC and decode_single_address(
_log['topics'][1]) == from_address:
has_transfer_event = True
break
if not has_transfer_event:
# there was no Transfer event matching this transaction, this means something went wrong
token_tx_status = 'error'
else:
erc20_dispatcher.update_token_cache(
token_tx['contract_address'],
from_address,
to_address,
blocknumber=parse_int(
transaction['blockNumber']))
else:
log.error(
"Unexpectedly got null for tx receipt logs for tx: {}"
.format(tx['hash']))
token_tx_status = 'error'
token_tx_updates.append(
(token_tx_status, tx['transaction_id'],
token_tx['transaction_log_index']))
token_tx = dict(token_tx)
token_tx['status'] = token_tx_status
updated_token_txs.append(token_tx)
token_txs = updated_token_txs
blocknumber = parse_int(transaction['blockNumber'])
async with self.db:
await self.db.execute(
"UPDATE transactions SET status = $1, blocknumber = $2, updated = (now() AT TIME ZONE 'utc') "
"WHERE transaction_id = $3", status, blocknumber,
transaction_id)
if token_tx_updates:
await self.db.executemany(
"UPDATE token_transactions SET status = $1 "
"WHERE transaction_id = $2 AND transaction_log_index = $3",
token_tx_updates)
await self.db.commit()
else:
# this is probably because the node hasn't caught up with the latest block yet, retry in a "bit" (but only retry up to 60 seconds)
if retry_start_time > 0 and time.time(
) - retry_start_time >= 60:
if transaction is None:
log.error(
"requested transaction {}'s status to be set to confirmed, but cannot find the transaction"
.format(tx['hash']))
else:
log.error(
"requested transaction {}'s status to be set to confirmed, but transaction is not confirmed on the node"
.format(tx['hash']))
return
await asyncio.sleep(random.random())
manager_dispatcher.update_transaction(
transaction_id,
status,
retry_start_time=retry_start_time or time.time())
return
else:
async with self.db:
await self.db.execute(
"UPDATE transactions SET status = $1, updated = (now() AT TIME ZONE 'utc') WHERE transaction_id = $2",
status, transaction_id)
await self.db.commit()
# render notification
# don't send "queued"
if status == 'queued':
status = 'unconfirmed'
elif status == 'unconfirmed' and tx['status'] == 'queued':
# there's already been a tx for this so no need to send another
return
messages = []
# check if this is an erc20 transaction, if so use those values
if token_txs:
for token_tx in token_txs:
token_tx_status = token_tx['status']
from_address = token_tx['from_address']
to_address = token_tx['to_address']
# TokenPayment PNs are not shown at the moment, so i'm removing
# this for the time being until they're required
# if token_tx_status == 'confirmed':
# data = {
# "txHash": tx['hash'],
# "fromAddress": from_address,
# "toAddress": to_address,
# "status": token_tx_status,
# "value": token_tx['value'],
# "contractAddress": token_tx['contract_address']
# }
# messages.append((from_address, to_address, token_tx_status, "SOFA::TokenPayment: " + json_encode(data)))
# if a WETH deposit or withdrawal, we need to let the client know to
# update their ETHER balance using a normal SOFA:Payment
if token_tx['contract_address'] == WETH_CONTRACT_ADDRESS and (
from_address
== "0x0000000000000000000000000000000000000000"
or to_address
== "0x0000000000000000000000000000000000000000"):
payment = SofaPayment(
value=parse_int(token_tx['value']),
txHash=tx['hash'],
status=status,
fromAddress=from_address,
toAddress=to_address,
networkId=config['ethereum']['network_id'])
messages.append(
(from_address, to_address, status, payment.render()))
else:
from_address = tx['from_address']
to_address = tx['to_address']
payment = SofaPayment(value=parse_int(tx['value']),
txHash=tx['hash'],
status=status,
fromAddress=from_address,
toAddress=to_address,
networkId=config['ethereum']['network_id'])
messages.append(
(from_address, to_address, status, payment.render()))
# figure out what addresses need pns
# from address always needs a pn
for from_address, to_address, status, message in messages:
manager_dispatcher.send_notification(from_address, message)
# no need to check to_address for contract deployments
if to_address == "0x":
# TODO: update any notification registrations to be marked as a contract
return
# check if this is a brand new tx with no status
if tx['status'] == 'new':
# if an error has happened before any PNs have been sent
# we only need to send the error to the sender, thus we
# only add 'to' if the new status is not an error
if status != 'error':
manager_dispatcher.send_notification(to_address, message)
else:
manager_dispatcher.send_notification(to_address, message)
# trigger a processing of the to_address's queue incase it has
# things waiting on this transaction
manager_dispatcher.process_transaction_queue(to_address)
async def sanity_check(self, frequency):
async with self.db:
rows = await self.db.fetch(
"SELECT DISTINCT from_address FROM transactions WHERE (status = 'unconfirmed' OR status = 'queued' OR status IS NULL) "
"AND v IS NOT NULL AND created < (now() AT TIME ZONE 'utc') - interval '3 minutes'"
)
rows2 = await self.db.fetch(
"WITH t1 AS (SELECT DISTINCT from_address FROM transactions WHERE (status IS NULL OR status = 'queued') AND v IS NOT NULL), "
"t2 AS (SELECT from_address, COUNT(*) FROM transactions WHERE (status = 'unconfirmed' AND v IS NOT NULL) GROUP BY from_address) "
"SELECT t1.from_address FROM t1 LEFT JOIN t2 ON t1.from_address = t2.from_address WHERE t2.count IS NULL;"
)
if rows or rows2:
log.debug(
"sanity check found {} addresses with potential problematic transactions"
.format(len(rows) + len(rows2)))
rows = set([row['from_address'] for row in rows
]).union(set([row['from_address'] for row in rows2]))
addresses_to_check = set()
old_and_unconfirmed = []
for ethereum_address in rows:
# check on unconfirmed transactions
async with self.db:
unconfirmed_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND status = 'unconfirmed' AND v IS NOT NULL",
ethereum_address)
if len(unconfirmed_transactions) > 0:
for transaction in unconfirmed_transactions:
# check on unconfirmed transactions first
if transaction['status'] == 'unconfirmed':
# we need to check the true status of unconfirmed transactions
# as the block monitor may be inbetween calls and not have seen
# this transaction to mark it as confirmed.
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
# sanity check to make sure the tx still exists
if tx is None:
# if not, set to error!
log.warning(
"WARNING: unconfirmed transaction '{}' is not visible on the monitor node. Setting back to queued to force resubmission"
.format(transaction['hash']))
async with self.db:
await self.db.execute(
"UPDATE transactions SET status = $2 WHERE transaction_id = $1",
transaction['transaction_id'], 'queued')
await self.db.commit()
addresses_to_check.add(transaction['from_address'])
elif tx['blockNumber'] is not None:
# confirmed! update the status
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(transaction['from_address'])
addresses_to_check.add(transaction['to_address'])
else:
old_and_unconfirmed.append(transaction['hash'])
else:
# make sure there are pending incoming transactions
async with self.db:
incoming_transactions = await self.db.fetchrow(
"SELECT 1 FROM transactions "
"WHERE to_address = $1 "
"AND (status = 'unconfirmed' OR status = 'queued')",
ethereum_address)
if not incoming_transactions:
log.error(
"ERROR: {} has transactions in it's queue, but no unconfirmed transactions!"
.format(ethereum_address))
# trigger queue processing as last resort
addresses_to_check.add(ethereum_address)
if len(old_and_unconfirmed):
log.warning(
"WARNING: {} transactions are old and unconfirmed!".format(
len(old_and_unconfirmed)))
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
manager_dispatcher.process_transaction_queue(address)
if frequency:
manager_dispatcher.sanity_check(frequency).delay(frequency)
async def update_transaction(self, transaction_id, status):
async with self.db:
tx = await self.db.fetchrow(
"SELECT * FROM transactions WHERE transaction_id = $1",
transaction_id)
if tx is None or tx['status'] == status:
return
token_txs = await self.db.fetch(
"SELECT tok.symbol, tok.name, tok.decimals, tx.contract_address, tx.value, tx.from_address, tx.to_address, tx.transaction_log_index, tx.status "
"FROM token_transactions tx "
"JOIN tokens tok "
"ON tok.contract_address = tx.contract_address "
"WHERE tx.transaction_id = $1", transaction_id)
# check if we're trying to update the state of a tx that is already confirmed, we have an issue
if tx['status'] == 'confirmed':
log.warning(
"Trying to update status of tx {} to {}, but tx is already confirmed"
.format(tx['hash'], status))
return
# only log if the transaction is internal
if tx['v'] is not None:
log.info(
"Updating status of tx {} to {} (previously: {})".format(
tx['hash'], status, tx['status']))
if status == 'confirmed':
transaction = await self.eth.eth_getTransactionByHash(tx['hash'])
if transaction and 'blockNumber' in transaction:
blocknumber = parse_int(transaction['blockNumber'])
async with self.db:
await self.db.execute(
"UPDATE transactions SET status = $1, blocknumber = $2, updated = (now() AT TIME ZONE 'utc') "
"WHERE transaction_id = $3", status, blocknumber,
transaction_id)
await self.db.commit()
else:
log.error(
"requested transaction '{}''s status to be set to confirmed, but cannot find the transaction"
.format(tx['hash']))
else:
async with self.db:
await self.db.execute(
"UPDATE transactions SET status = $1, updated = (now() AT TIME ZONE 'utc') WHERE transaction_id = $2",
status, transaction_id)
await self.db.commit()
# render notification
# don't send "queued"
if status == 'queued':
status = 'unconfirmed'
elif status == 'unconfirmed' and tx['status'] == 'queued':
# there's already been a tx for this so no need to send another
return
messages = []
# check if this is an erc20 transaction, if so use those values
if token_txs:
if status == 'confirmed':
tx_receipt = await self.eth.eth_getTransactionReceipt(
tx['hash'])
if tx_receipt is None:
log.error(
"Failed to get transaction receipt for confirmed transaction: {}"
.format(tx_receipt))
# requeue to try again
manager_dispatcher.update_transaction(
transaction_id, status)
return
for token_tx in token_txs:
token_tx_status = status
from_address = token_tx['from_address']
to_address = token_tx['to_address']
if status == 'confirmed':
# check transaction receipt to make sure the transfer was successful
has_transfer_event = False
if tx_receipt[
'logs'] is not None: # should always be [], but checking just incase
for _log in tx_receipt['logs']:
if len(_log['topics']) > 2:
if _log['topics'][0] == TRANSFER_TOPIC and \
decode_single_address(_log['topics'][1]) == from_address and \
decode_single_address(_log['topics'][2]) == to_address:
has_transfer_event = True
break
elif _log['address'] == WETH_CONTRACT_ADDRESS:
if _log['topics'][
0] == DEPOSIT_TOPIC and decode_single_address(
_log['topics'][1]) == to_address:
has_transfer_event = True
break
elif _log['topics'][
0] == WITHDRAWAL_TOPIC and decode_single_address(
_log['topics'][1]) == from_address:
has_transfer_event = True
break
if not has_transfer_event:
# there was no Transfer event matching this transaction
token_tx_status = 'error'
else:
erc20_dispatcher.update_token_cache(
token_tx['contract_address'],
from_address,
to_address,
blocknumber=parse_int(transaction['blockNumber']))
if token_tx_status == 'confirmed':
data = {
"txHash": tx['hash'],
"fromAddress": from_address,
"toAddress": to_address,
"status": token_tx_status,
"value": token_tx['value'],
"contractAddress": token_tx['contract_address']
}
messages.append(
(from_address, to_address, token_tx_status,
"SOFA::TokenPayment: " + json_encode(data)))
async with self.db:
await self.db.execute(
"UPDATE token_transactions SET status = $1 "
"WHERE transaction_id = $2 AND transaction_log_index = $3",
token_tx_status, tx['transaction_id'],
token_tx['transaction_log_index'])
await self.db.commit()
# if a WETH deposit or withdrawal, we need to let the client know to
# update their ETHER balance using a normal SOFA:Payment
if token_tx['contract_address'] == WETH_CONTRACT_ADDRESS and (
from_address
== "0x0000000000000000000000000000000000000000"
or to_address
== "0x0000000000000000000000000000000000000000"):
payment = SofaPayment(
value=parse_int(token_tx['value']),
txHash=tx['hash'],
status=status,
fromAddress=from_address,
toAddress=to_address,
networkId=config['ethereum']['network_id'])
messages.append(
(from_address, to_address, status, payment.render()))
else:
from_address = tx['from_address']
to_address = tx['to_address']
payment = SofaPayment(value=parse_int(tx['value']),
txHash=tx['hash'],
status=status,
fromAddress=from_address,
toAddress=to_address,
networkId=config['ethereum']['network_id'])
messages.append(
(from_address, to_address, status, payment.render()))
# figure out what addresses need pns
# from address always needs a pn
for from_address, to_address, status, message in messages:
manager_dispatcher.send_notification(from_address, message)
# no need to check to_address for contract deployments
if to_address == "0x":
# TODO: update any notification registrations to be marked as a contract
return
# check if this is a brand new tx with no status
if tx['status'] is None:
# if an error has happened before any PNs have been sent
# we only need to send the error to the sender, thus we
# only add 'to' if the new status is not an error
if status != 'error':
manager_dispatcher.send_notification(to_address, message)
else:
manager_dispatcher.send_notification(to_address, message)
# trigger a processing of the to_address's queue incase it has
# things waiting on this transaction
manager_dispatcher.process_transaction_queue(to_address)