async def test_cancel_transaction_fails_if_not_queued(self):
tx = create_transaction(nonce=0, gasprice=10 ** 10, startgas=21000, to=TEST_ADDRESS, value=10 ** 18)
tx = sign_transaction(tx, FAUCET_PRIVATE_KEY)
tx_hash = calculate_transaction_hash(tx)
from_address = FAUCET_ADDRESS
to_address = TEST_ADDRESS
async with self.pool.acquire() as con:
await con.execute("INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, data, v, r, s, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
tx_hash, from_address, to_address, tx.nonce,
hex(tx.value), hex(tx.startgas), hex(tx.gasprice),
data_encoder(tx.data), hex(tx.v), hex(tx.r), hex(tx.s),
'unconfirmed')
signature = personal_sign(FAUCET_PRIVATE_KEY, "Cancel transaction " + tx_hash)
resp = await self.fetch("/tx/cancel", method="POST", body={"tx_hash": tx_hash, "signature": signature})
self.assertResponseCodeEqual(resp, 400)
async with self.pool.acquire() as con:
await con.execute("UPDATE transactions SET status = 'confirmed'")
resp = await self.fetch("/tx/cancel", method="POST", body={"tx_hash": tx_hash, "signature": signature})
self.assertResponseCodeEqual(resp, 400)
async with self.pool.acquire() as con:
await con.execute("UPDATE transactions SET status = 'error'")
resp = await self.fetch("/tx/cancel", method="POST", body={"tx_hash": tx_hash, "signature": signature})
self.assertResponseCodeEqual(resp, 400)
async def test_cancel_transaction(self):
listener = MockTaskListener(self._app)
listener.start_task_listener()
tx = create_transaction(nonce=0, gasprice=10 ** 10, startgas=21000, to=TEST_ADDRESS, value=10 ** 18)
tx = sign_transaction(tx, FAUCET_PRIVATE_KEY)
tx_hash = calculate_transaction_hash(tx)
from_address = FAUCET_ADDRESS
to_address = TEST_ADDRESS
async with self.pool.acquire() as con:
await con.execute("INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, data, v, r, s, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
tx_hash, from_address, to_address, tx.nonce,
hex(tx.value), hex(tx.startgas), hex(tx.gasprice),
data_encoder(tx.data), hex(tx.v), hex(tx.r), hex(tx.s),
'queued')
signature = personal_sign(FAUCET_PRIVATE_KEY, "Cancel transaction " + tx_hash)
resp = await self.fetch("/tx/cancel", method="POST", body={"tx_hash": tx_hash, "signature": signature})
self.assertResponseCodeEqual(resp, 204)
tx_id, status = await listener.get()
self.assertEqual(tx_id, 1)
self.assertEqual(status, 'error')
await listener.stop_task_listener()
async def send_transaction(self, *, tx, signature=None):
try:
tx = decode_transaction(tx)
except:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Invalid Transaction'
})
if is_transaction_signed(tx):
tx_sig = data_encoder(signature_from_transaction(tx))
if signature:
if tx_sig != signature:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: Signature in payload and signature of transaction do not match'
})
else:
signature = tx_sig
else:
if signature is None:
raise JsonRPCInvalidParamsError(data={
'id': 'missing_signature',
'message': 'Missing Signature'
})
if not validate_signature(signature):
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: {}'.format('Invalid length' if len(
signature) != 132 else 'Invalid hex value')
})
try:
sig = data_decoder(signature)
except Exception:
log.exception(
"Unexpected error decoding valid signature: {}".format(
signature))
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature'
})
add_signature_to_transaction(tx, sig)
# validate network id, if it's not for "all networks"
if tx.network_id is not None and self.network_id != tx.network_id:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_network_id',
'message': 'Invalid Network ID'
})
from_address = data_encoder(tx.sender)
to_address = data_encoder(tx.to)
# prevent spamming of transactions with the same nonce from the same sender
with RedisLock(self.redis,
"{}:{}".format(from_address, tx.nonce),
raise_when_locked=partial(JsonRPCInvalidParamsError,
data={
'id':
'invalid_nonce',
'message':
'Nonce already used'
}),
ex=5):
# disallow transaction overwriting for known transactions
async with self.db:
existing = await self.db.fetchrow(
"SELECT * FROM transactions WHERE "
"from_address = $1 AND nonce = $2 AND status != $3",
from_address, tx.nonce, 'error')
if existing:
# debugging checks
existing_tx = await self.eth.eth_getTransactionByHash(
existing['hash'])
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_nonce',
'message': 'Nonce already used'
})
# make sure the account has enough funds for the transaction
network_balance, balance, _, _ = await self.get_balances(
from_address)
#log.info("Attempting to send transaction\nHash: {}\n{} -> {}\nValue: {} + {} (gas) * {} (startgas) = {}\nSender's Balance {} ({} unconfirmed)".format(
# calculate_transaction_hash(tx), from_address, to_address, tx.value, tx.startgas, tx.gasprice, tx.value + (tx.startgas * tx.gasprice), network_balance, balance))
if balance < (tx.value + (tx.startgas * tx.gasprice)):
raise JsonRPCInsufficientFundsError(
data={
'id': 'insufficient_funds',
'message': 'Insufficient Funds'
})
# validate the nonce
c_nonce = await self.get_transaction_count(from_address)
if tx.nonce < c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too low'
})
if tx.nonce > c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too high'
})
if tx.intrinsic_gas_used > tx.startgas:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_transaction',
'message':
'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'
.format(tx.intrinsic_gas_used, tx.startgas)
})
# now this tx fits enough of the criteria to allow it
# onto the transaction queue
tx_hash = calculate_transaction_hash(tx)
# add tx to database
async with self.db:
await self.db.execute(
"INSERT INTO transactions "
"(hash, from_address, to_address, nonce, "
"value, gas, gas_price, "
"data, v, r, s, "
"sender_toshi_id) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
tx_hash, from_address, to_address, tx.nonce, hex(tx.value),
hex(tx.startgas), hex(tx.gasprice), data_encoder(tx.data),
hex(tx.v), hex(tx.r), hex(tx.s), self.user_toshi_id)
await self.db.commit()
# trigger processing the transaction queue
self.tasks.process_transaction_queue(from_address)
# analytics
# use notification registrations to try find toshi ids for users
if self.user_toshi_id:
sender_toshi_id = self.user_toshi_id
else:
async with self.db:
sender_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", from_address)
async with self.db:
receiver_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", to_address)
self.track(sender_toshi_id, "Sent transaction")
# it doesn't make sense to add user agent here as we
# don't know the receiver's user agent
self.track(receiver_toshi_id,
"Received transaction",
add_user_agent=False)
return tx_hash
async def send_transaction(self, *, tx, signature=None):
try:
tx = decode_transaction(tx)
except:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Invalid Transaction'
})
if is_transaction_signed(tx):
tx_sig = data_encoder(signature_from_transaction(tx))
if signature:
if tx_sig != signature:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: Signature in payload and signature of transaction do not match'
})
else:
signature = tx_sig
else:
if signature is None:
raise JsonRPCInvalidParamsError(data={
'id': 'missing_signature',
'message': 'Missing Signature'
})
if not validate_signature(signature):
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: {}'.format('Invalid length' if len(
signature) != 132 else 'Invalid hex value')
})
try:
sig = data_decoder(signature)
except Exception:
log.exception(
"Unexpected error decoding valid signature: {}".format(
signature))
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature'
})
add_signature_to_transaction(tx, sig)
# validate network id, if it's not for "all networks"
if tx.network_id is not None and self.network_id != tx.network_id:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_network_id',
'message': 'Invalid Network ID'
})
from_address = data_encoder(tx.sender)
to_address = data_encoder(tx.to)
# prevent spamming of transactions with the same nonce from the same sender
async with RedisLock("{}:{}".format(from_address, tx.nonce),
raise_when_locked=partial(
JsonRPCInvalidParamsError,
data={
'id': 'invalid_nonce',
'message': 'Nonce already used'
}),
ex=5):
# check for transaction overwriting
async with self.db:
existing = await self.db.fetchrow(
"SELECT * FROM transactions WHERE "
"from_address = $1 AND nonce = $2 AND "
"(status != 'error' or status is NULL)", from_address,
tx.nonce)
# disallow transaction overwriting when the gas is lower or the transaction is confirmed
if existing and (parse_int(existing['gas_price']) >= tx.gasprice
or existing['status'] == 'confirmed'):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_nonce',
'message': 'Nonce already used'
})
# make sure the account has enough funds for the transaction
network_balance, balance, _, _ = await self.get_balances(
from_address)
if existing:
balance += parse_int(existing['value']) + parse_int(
existing['gas']) * parse_int(existing['gas_price'])
if balance < (tx.value + (tx.startgas * tx.gasprice)):
raise JsonRPCInsufficientFundsError(
data={
'id': 'insufficient_funds',
'message': 'Insufficient Funds'
})
# validate the nonce (only necessary if tx doesn't already exist)
if not existing:
c_nonce = await self.get_transaction_count(from_address)
if tx.nonce < c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too low'
})
if tx.nonce > c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too high'
})
if tx.intrinsic_gas_used > tx.startgas:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_transaction',
'message':
'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'
.format(tx.intrinsic_gas_used, tx.startgas)
})
# now this tx fits enough of the criteria to allow it
# onto the transaction queue
tx_hash = calculate_transaction_hash(tx)
if existing:
log.info(
"Setting tx '{}' to error due to forced overwrite".format(
existing['hash']))
manager_dispatcher.update_transaction(
existing['transaction_id'], 'error')
data = data_encoder(tx.data)
if data and \
((data.startswith("0xa9059cbb") and len(data) == 138) or \
(data.startswith("0x23b872dd") and len(data) == 202)):
# check if the token is a known erc20 token
async with self.db:
erc20_token = await self.db.fetchrow(
"SELECT * FROM tokens WHERE contract_address = $1",
to_address)
else:
erc20_token = False
# add tx to database
async with self.db:
db_tx = await self.db.fetchrow(
"INSERT INTO transactions "
"(hash, from_address, to_address, nonce, "
"value, gas, gas_price, "
"data, v, r, s, "
"sender_toshi_id) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) "
"RETURNING transaction_id", tx_hash, from_address,
to_address, tx.nonce, hex(tx.value), hex(tx.startgas),
hex(tx.gasprice), data_encoder(tx.data), hex(tx.v),
hex(tx.r), hex(tx.s), self.user_toshi_id)
if erc20_token:
token_value = int(data[-64:], 16)
if data.startswith("0x23b872dd"):
erc20_from_address = "0x" + data[34:74]
erc20_to_address = "0x" + data[98:138]
else:
erc20_from_address = from_address
erc20_to_address = "0x" + data[34:74]
await self.db.execute(
"INSERT INTO token_transactions "
"(transaction_id, transaction_log_index, contract_address, from_address, to_address, value) "
"VALUES ($1, $2, $3, $4, $5, $6)",
db_tx['transaction_id'], 0,
erc20_token['contract_address'], erc20_from_address,
erc20_to_address, hex(token_value))
await self.db.commit()
# trigger processing the transaction queue
manager_dispatcher.process_transaction_queue(from_address)
# analytics
# use notification registrations to try find toshi ids for users
if self.user_toshi_id:
sender_toshi_id = self.user_toshi_id
else:
async with self.db:
sender_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", from_address)
async with self.db:
receiver_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", to_address)
self.track(sender_toshi_id, "Sent transaction")
# it doesn't make sense to add user agent here as we
# don't know the receiver's user agent
self.track(receiver_toshi_id,
"Received transaction",
add_user_agent=False)
return tx_hash
async def sanity_check(self, frequency):
async with self.db:
rows = await self.db.fetch(
"SELECT DISTINCT from_address FROM transactions WHERE (status = 'unconfirmed' OR status = 'queued' OR status = 'new') "
"AND v IS NOT NULL AND created < (now() AT TIME ZONE 'utc') - interval '3 minutes'"
)
rows2 = await self.db.fetch(
"WITH t1 AS (SELECT DISTINCT from_address FROM transactions WHERE (status = 'new' OR status = 'queued') AND v IS NOT NULL), "
"t2 AS (SELECT from_address, COUNT(*) FROM transactions WHERE (status = 'unconfirmed' AND v IS NOT NULL) GROUP BY from_address) "
"SELECT t1.from_address FROM t1 LEFT JOIN t2 ON t1.from_address = t2.from_address WHERE t2.count IS NULL;"
)
if rows or rows2:
log.debug(
"sanity check found {} addresses with potential problematic transactions"
.format(len(rows) + len(rows2)))
rows = set([row['from_address'] for row in rows
]).union(set([row['from_address'] for row in rows2]))
addresses_to_check = set()
old_and_unconfirmed = []
for ethereum_address in rows:
# check on queued transactions
async with self.db:
queued_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'new' OR status = 'queued') AND v IS NOT NULL",
ethereum_address)
if queued_transactions:
# make sure there are pending incoming transactions
async with self.db:
incoming_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE to_address = $1 "
"AND (status = 'unconfirmed' OR status = 'queued' OR status = 'new')",
ethereum_address)
if not incoming_transactions:
log.error(
"ERROR: {} has transactions in it's queue, but no unconfirmed transactions!"
.format(ethereum_address))
# trigger queue processing as last resort
addresses_to_check.add(ethereum_address)
else:
# check health of the incoming transaction
for transaction in incoming_transactions:
if transaction['v'] is None:
try:
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
except:
log.exception(
"Error getting transaction {} in sanity check",
transaction['hash'])
continue
if tx is None:
log.warning(
"external transaction (id: {}) no longer found on nodes"
.format(transaction['transaction_id']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(ethereum_address)
elif tx['blockNumber'] is not None:
log.warning(
"external transaction (id: {}) confirmed on node, but wasn't confirmed in db"
.format(transaction['transaction_id']))
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(ethereum_address)
# no need to continue with dealing with unconfirmed transactions if there are queued ones
continue
async with self.db:
unconfirmed_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND status = 'unconfirmed' AND v IS NOT NULL",
ethereum_address)
if unconfirmed_transactions:
for transaction in unconfirmed_transactions:
# check on unconfirmed transactions first
if transaction['status'] == 'unconfirmed':
# we neehed to check the true status of unconfirmed transactions
# as the block monitor may be inbetween calls and not have seen
# this transaction to mark it as confirmed.
try:
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
except:
log.exception(
"Error getting transaction {} in sanity check",
transaction['hash'])
continue
# sanity check to make sure the tx still exists
if tx is None:
# if not, try resubmit
# NOTE: it may just be an issue with load balanced nodes not seeing all pending transactions
# so we don't want to adjust the status of the transaction at all at this stage
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
data = data_decoder(
transaction['data']
) if transaction['data'] else b''
tx = create_transaction(
nonce=transaction['nonce'],
value=value,
gasprice=gas_price,
startgas=gas,
to=transaction['to_address'],
data=data,
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
if calculate_transaction_hash(
tx) != transaction['hash']:
log.warning(
"error resubmitting transaction {}: regenerating tx resulted in a different hash"
.format(transaction['hash']))
else:
tx_encoded = encode_transaction(tx)
try:
await self.eth.eth_sendRawTransaction(
tx_encoded)
addresses_to_check.add(
transaction['from_address'])
except Exception as e:
# note: usually not critical, don't panic
log.warning(
"error resubmitting transaction {}: {}"
.format(transaction['hash'], str(e)))
elif tx['blockNumber'] is not None:
# confirmed! update the status
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(transaction['from_address'])
addresses_to_check.add(transaction['to_address'])
else:
old_and_unconfirmed.append(transaction['hash'])
if len(old_and_unconfirmed):
log.warning(
"WARNING: {} transactions are old and unconfirmed!".format(
len(old_and_unconfirmed)))
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
manager_dispatcher.process_transaction_queue(address)
if frequency:
manager_dispatcher.sanity_check(frequency).delay(frequency)
