def start_interval_services(self):
manager_dispatcher.sanity_check(60).delay(60)
def start_interval_services(self):
manager_dispatcher.sanity_check(60).delay(60)
manager_dispatcher.update_default_gas_price(60).delay(60)
async def sanity_check(self, frequency):
async with self.db:
rows = await self.db.fetch(
"SELECT DISTINCT from_address FROM transactions WHERE (status = 'unconfirmed' OR status = 'queued' OR status = 'new') "
"AND v IS NOT NULL AND created < (now() AT TIME ZONE 'utc') - interval '3 minutes'"
)
rows2 = await self.db.fetch(
"WITH t1 AS (SELECT DISTINCT from_address FROM transactions WHERE (status = 'new' OR status = 'queued') AND v IS NOT NULL), "
"t2 AS (SELECT from_address, COUNT(*) FROM transactions WHERE (status = 'unconfirmed' AND v IS NOT NULL) GROUP BY from_address) "
"SELECT t1.from_address FROM t1 LEFT JOIN t2 ON t1.from_address = t2.from_address WHERE t2.count IS NULL;"
)
if rows or rows2:
log.debug(
"sanity check found {} addresses with potential problematic transactions"
.format(len(rows) + len(rows2)))
rows = set([row['from_address'] for row in rows
]).union(set([row['from_address'] for row in rows2]))
addresses_to_check = set()
old_and_unconfirmed = []
for ethereum_address in rows:
# check on queued transactions
async with self.db:
queued_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'new' OR status = 'queued') AND v IS NOT NULL",
ethereum_address)
if queued_transactions:
# make sure there are pending incoming transactions
async with self.db:
incoming_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE to_address = $1 "
"AND (status = 'unconfirmed' OR status = 'queued' OR status = 'new')",
ethereum_address)
if not incoming_transactions:
log.error(
"ERROR: {} has transactions in it's queue, but no unconfirmed transactions!"
.format(ethereum_address))
# trigger queue processing as last resort
addresses_to_check.add(ethereum_address)
else:
# check health of the incoming transaction
for transaction in incoming_transactions:
if transaction['v'] is None:
try:
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
except:
log.exception(
"Error getting transaction {} in sanity check",
transaction['hash'])
continue
if tx is None:
log.warning(
"external transaction (id: {}) no longer found on nodes"
.format(transaction['transaction_id']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(ethereum_address)
elif tx['blockNumber'] is not None:
log.warning(
"external transaction (id: {}) confirmed on node, but wasn't confirmed in db"
.format(transaction['transaction_id']))
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(ethereum_address)
# no need to continue with dealing with unconfirmed transactions if there are queued ones
continue
async with self.db:
unconfirmed_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND status = 'unconfirmed' AND v IS NOT NULL",
ethereum_address)
if unconfirmed_transactions:
for transaction in unconfirmed_transactions:
# check on unconfirmed transactions first
if transaction['status'] == 'unconfirmed':
# we neehed to check the true status of unconfirmed transactions
# as the block monitor may be inbetween calls and not have seen
# this transaction to mark it as confirmed.
try:
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
except:
log.exception(
"Error getting transaction {} in sanity check",
transaction['hash'])
continue
# sanity check to make sure the tx still exists
if tx is None:
# if not, try resubmit
# NOTE: it may just be an issue with load balanced nodes not seeing all pending transactions
# so we don't want to adjust the status of the transaction at all at this stage
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
data = data_decoder(
transaction['data']
) if transaction['data'] else b''
tx = create_transaction(
nonce=transaction['nonce'],
value=value,
gasprice=gas_price,
startgas=gas,
to=transaction['to_address'],
data=data,
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
if calculate_transaction_hash(
tx) != transaction['hash']:
log.warning(
"error resubmitting transaction {}: regenerating tx resulted in a different hash"
.format(transaction['hash']))
else:
tx_encoded = encode_transaction(tx)
try:
await self.eth.eth_sendRawTransaction(
tx_encoded)
addresses_to_check.add(
transaction['from_address'])
except Exception as e:
# note: usually not critical, don't panic
log.warning(
"error resubmitting transaction {}: {}"
.format(transaction['hash'], str(e)))
elif tx['blockNumber'] is not None:
# confirmed! update the status
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(transaction['from_address'])
addresses_to_check.add(transaction['to_address'])
else:
old_and_unconfirmed.append(transaction['hash'])
if len(old_and_unconfirmed):
log.warning(
"WARNING: {} transactions are old and unconfirmed!".format(
len(old_and_unconfirmed)))
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
manager_dispatcher.process_transaction_queue(address)
if frequency:
manager_dispatcher.sanity_check(frequency).delay(frequency)
async def sanity_check(self, frequency):
async with self.db:
rows = await self.db.fetch(
"SELECT DISTINCT from_address FROM transactions WHERE (status = 'unconfirmed' OR status = 'queued' OR status IS NULL) "
"AND v IS NOT NULL AND created < (now() AT TIME ZONE 'utc') - interval '3 minutes'"
)
rows2 = await self.db.fetch(
"WITH t1 AS (SELECT DISTINCT from_address FROM transactions WHERE (status IS NULL OR status = 'queued') AND v IS NOT NULL), "
"t2 AS (SELECT from_address, COUNT(*) FROM transactions WHERE (status = 'unconfirmed' AND v IS NOT NULL) GROUP BY from_address) "
"SELECT t1.from_address FROM t1 LEFT JOIN t2 ON t1.from_address = t2.from_address WHERE t2.count IS NULL;"
)
if rows or rows2:
log.debug(
"sanity check found {} addresses with potential problematic transactions"
.format(len(rows) + len(rows2)))
rows = set([row['from_address'] for row in rows
]).union(set([row['from_address'] for row in rows2]))
addresses_to_check = set()
old_and_unconfirmed = []
for ethereum_address in rows:
# check on unconfirmed transactions
async with self.db:
unconfirmed_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND status = 'unconfirmed' AND v IS NOT NULL",
ethereum_address)
if len(unconfirmed_transactions) > 0:
for transaction in unconfirmed_transactions:
# check on unconfirmed transactions first
if transaction['status'] == 'unconfirmed':
# we need to check the true status of unconfirmed transactions
# as the block monitor may be inbetween calls and not have seen
# this transaction to mark it as confirmed.
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
# sanity check to make sure the tx still exists
if tx is None:
# if not, set to error!
log.warning(
"WARNING: unconfirmed transaction '{}' is not visible on the monitor node. Setting back to queued to force resubmission"
.format(transaction['hash']))
async with self.db:
await self.db.execute(
"UPDATE transactions SET status = $2 WHERE transaction_id = $1",
transaction['transaction_id'], 'queued')
await self.db.commit()
addresses_to_check.add(transaction['from_address'])
elif tx['blockNumber'] is not None:
# confirmed! update the status
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(transaction['from_address'])
addresses_to_check.add(transaction['to_address'])
else:
old_and_unconfirmed.append(transaction['hash'])
else:
# make sure there are pending incoming transactions
async with self.db:
incoming_transactions = await self.db.fetchrow(
"SELECT 1 FROM transactions "
"WHERE to_address = $1 "
"AND (status = 'unconfirmed' OR status = 'queued')",
ethereum_address)
if not incoming_transactions:
log.error(
"ERROR: {} has transactions in it's queue, but no unconfirmed transactions!"
.format(ethereum_address))
# trigger queue processing as last resort
addresses_to_check.add(ethereum_address)
if len(old_and_unconfirmed):
log.warning(
"WARNING: {} transactions are old and unconfirmed!".format(
len(old_and_unconfirmed)))
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
manager_dispatcher.process_transaction_queue(address)
if frequency:
manager_dispatcher.sanity_check(frequency).delay(frequency)