def test_add_signature_to_transaction_with_netowrk_id(self):
for network_id in [1, 2, 66, 100]:
sender_private_key = "0x0164f7c7399f4bb1eafeaae699ebbb12050bc6a50b2836b9ca766068a9d000c0"
sender_address = "0xde3d2d9dd52ea80f7799ef4791063a5458d13913"
to_address = "0x056db290f8ba3250ca64a45d16284d04bc6f5fbf"
value = 10000000000
nonce = 1048576
data = b''
gasprice = DEFAULT_GASPRICE
startgas = DEFAULT_STARTGAS
network_id = 1
tx1 = Transaction(nonce, gasprice, startgas, to_address, value,
data, network_id, 0, 0)
tx = encode_transaction(tx1)
tx1.sign(data_decoder(sender_private_key), network_id=network_id)
expected_signed_tx = encode_transaction(tx1)
sig = data_encoder(signature_from_transaction(tx1))
signed_tx = add_signature_to_transaction(tx, sig)
self.assertEqual(signed_tx, expected_signed_tx)
tx_obj = decode_transaction(tx)
add_signature_to_transaction(tx_obj, sig)
self.assertEqual(tx_obj.network_id, network_id)
self.assertEqual(data_encoder(tx_obj.sender), sender_address)
self.assertEqual(encode_transaction(tx_obj), expected_signed_tx)
async def test_create_and_send_signed_transaction_with_separate_sig(self):
body = {"from": FAUCET_ADDRESS, "to": TEST_ADDRESS, "value": 10**10}
resp = await self.fetch("/tx/skel", method="POST", body=body)
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx = decode_transaction(body['tx'])
tx = sign_transaction(tx, FAUCET_PRIVATE_KEY)
sig = signature_from_transaction(tx)
body = {"tx": encode_transaction(tx), "signature": data_encoder(sig)}
resp = await self.fetch("/tx", method="POST", body=body)
self.assertEqual(resp.code, 200, resp.body)
body = json_decode(resp.body)
tx_hash = body['tx_hash']
async def check_db():
async with self.pool.acquire() as con:
rows = await con.fetch(
"SELECT * FROM transactions WHERE nonce = $1", tx.nonce)
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0]['hash'], tx_hash)
await self.wait_on_tx_confirmation(tx_hash)
async def test_transactions_get_queued_in_queue_is_weird(self, *, monitor, manager):
"""This test reproduces a bug where if there was a transaction with too low a gas price queued,
the monitor sanity check would force any further queued transactions with a good gas price and
cause the previous transaction to the error state due to the nonce being problematic.
This tests that even if the queue gets in a weird state like this it still gets resolved correctly
"""
gas_price = 50000000000
safe_low_gas_price = 40000000000
too_low_gas_price = 30000000000
assert(gas_price != DEFAULT_GASPRICE)
assert(gas_price > safe_low_gas_price)
assert(safe_low_gas_price > too_low_gas_price)
await self.redis.set("gas_station_standard_gas_price", hex(gas_price))
await self.redis.set("gas_station_safelow_gas_price", hex(safe_low_gas_price))
for nonce, gasprice, value in [(0, too_low_gas_price, 0x1000000000000),
(1, safe_low_gas_price, 0x2000000000000),
(2, gas_price, 0x3000000000000)]:
tx = create_transaction(nonce=0x100000 + nonce, gasprice=gasprice, startgas=DEFAULT_STARTGAS,
to=TEST_ADDRESS, value=value, network_id=0x42)
await self.sign_and_send_tx(FAUCET_PRIVATE_KEY, encode_transaction(tx))
await asyncio.sleep(0.1)
async with self.pool.acquire() as con:
txs = await con.fetch("SELECT * FROM transactions")
for tx in txs:
self.assertEqual(tx['status'], 'queued')
# purposefully cause some problems
async with self.pool.acquire() as con:
txs = await con.fetch("UPDATE transactions SET status = 'unconfirmed' WHERE nonce > $1", 0x100000)
from toshieth.tasks import manager_dispatcher
manager_dispatcher.process_transaction_queue(FAUCET_ADDRESS)
await asyncio.sleep(0.1)
async with self.pool.acquire() as con:
tx = await con.fetchrow("SELECT * FROM transactions WHERE nonce = $1", 0x100000)
self.assertEqual(tx['status'], 'queued')
# fix up gas prices
await self.redis.set("gas_station_standard_gas_price", hex(safe_low_gas_price))
await self.redis.set("gas_station_safelow_gas_price", hex(too_low_gas_price))
manager_dispatcher.process_transaction_queue(FAUCET_ADDRESS)
await self.wait_on_tx_confirmation(tx['hash'])
async with self.pool.acquire() as con:
tx = await con.fetchrow("SELECT * FROM transactions WHERE nonce = $1", 0x100000)
self.assertEqual(tx['status'], 'confirmed')
def send_tx(self, tx, signature=None, **kwargs):
if isinstance(tx, (Transaction, UnsignedTransaction)):
tx = encode_transaction(tx)
body = {"tx": tx}
if signature:
body['signature'] = signature
resp = self._fetch("/v1/tx", "POST", body, **kwargs)
return resp['tx_hash']
async def test_get_balance(self, *, ethminer):
addr = '0x39bf9e501e61440b4b268d7b2e9aa2458dd201bb'
val = 8761751855997712
val2 = int(val / 2)
await self.wait_on_tx_confirmation(await self.faucet(TEST_ID_ADDRESS, val))
ws_con = await self.websocket_connect(TEST_ID_KEY)
result = await ws_con.call("get_balance", [TEST_ID_ADDRESS])
self.assertEqual(int(result['confirmed_balance'][2:], 16), val)
self.assertEqual(int(result['unconfirmed_balance'][2:], 16), val)
# make sure no blocks get mined for a bit
ethminer.pause()
tx = create_transaction(nonce=0x100000, gasprice=DEFAULT_GASPRICE, startgas=DEFAULT_STARTGAS,
to=addr, value=val2, network_id=self.network_id)
tx = sign_transaction(tx, TEST_ID_KEY)
tx = encode_transaction(tx)
await ws_con.call("subscribe", [addr])
tx_hash = await ws_con.call("send_transaction", {"tx": tx})
new_balance = val - (val2 + DEFAULT_STARTGAS * DEFAULT_GASPRICE)
result = await ws_con.call("get_balance", [TEST_ID_ADDRESS])
self.assertEqual(int(result['confirmed_balance'][2:], 16), val)
self.assertEqual(int(result['unconfirmed_balance'][2:], 16), new_balance)
# check for the unconfirmed notification
result = await ws_con.read()
self.assertIsNotNone(result)
payment = parse_sofa_message(result['params']['message'])
self.assertEqual(payment['txHash'], tx_hash)
self.assertEqual(payment['status'], 'unconfirmed')
# restart mining
ethminer.start()
result = await ws_con.read()
payment = parse_sofa_message(result['params']['message'])
self.assertEqual(payment['txHash'], tx_hash)
self.assertEqual(payment['status'], 'confirmed')
result = await ws_con.call("get_balance", [TEST_ID_ADDRESS])
self.assertEqual(int(result['confirmed_balance'][2:], 16), new_balance,
"int('{}', 16) != {}".format(result['confirmed_balance'], new_balance))
self.assertEqual(int(result['unconfirmed_balance'][2:], 16), new_balance,
"int('{}', 16) != {}".format(result['unconfirmed_balance'], new_balance))
def test_get_skel(self):
tx = self.service_client.generate_tx_skel(
"0x0004DE837Ea93edbE51c093f45212AB22b4B35fc",
"0xdb089a4f9a8c5f17040b4fc51647e942b5fc601d",
1000000000000000000,
timeout=5)
self.assertEqual(
encode_transaction(tx),
"0xe9808504a817c80082520894db089a4f9a8c5f17040b4fc51647e942b5fc601d880de0b6b3a764000080"
)
async def test_send_txs_for_all_networks(self):
tx = create_transaction(nonce=1048576, gasprice=20 * 10**9, startgas=21000,
to="0x3535353535353535353535353535353535353535",
value=10**18, data=b'')
sign_transaction(tx, FAUCET_PRIVATE_KEY)
resp = await self.fetch("/tx", method="POST", body={
"tx": encode_transaction(tx)
})
self.assertEqual(resp.code, 200, resp.body)
await self.wait_on_tx_confirmation(data_encoder(tx.hash))
async def test_unable_to_send_txs_for_other_networks(self):
network_id = 1 # Test network id is 66
tx = create_transaction(nonce=9, gasprice=20 * 10**9, startgas=21000,
to="0x3535353535353535353535353535353535353535",
value=10**18, data=b'', network_id=network_id)
sign_transaction(tx, FAUCET_PRIVATE_KEY)
resp = await self.fetch("/tx", method="POST", body={
"tx": encode_transaction(tx)
})
self.assertEqual(resp.code, 400, resp.body)
def test_add_signature_to_transaction(self):
tx = "0xe9808504a817c80082520894db089a4f9a8c5f17040b4fc51647e942b5fc601d880de0b6b3a764000080"
sig = "0xf5a43adea07d366ae420a5c75a5cae6c60d3e4aaa0b72c2f37fc387efd43d7fd30c4327f2dbd959f654857f58912129b09763329459d08e25547d895ae90fa0f01"
expected_signed_tx = "0xf86c808504a817c80082520894db089a4f9a8c5f17040b4fc51647e942b5fc601d880de0b6b3a7640000801ca0f5a43adea07d366ae420a5c75a5cae6c60d3e4aaa0b72c2f37fc387efd43d7fda030c4327f2dbd959f654857f58912129b09763329459d08e25547d895ae90fa0f"
signed_tx = add_signature_to_transaction(tx, sig)
self.assertEqual(signed_tx, expected_signed_tx)
tx_obj = decode_transaction(tx)
add_signature_to_transaction(tx_obj, sig)
self.assertEqual(encode_transaction(tx_obj), expected_signed_tx)
async def create_transaction_skeleton(self,
*,
to_address,
from_address,
value=0,
nonce=None,
gas=None,
gas_price=None,
data=None):
if not validate_address(from_address):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_from_address',
'message': 'Invalid From Address'
})
if to_address is not None and not validate_address(to_address):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_to_address',
'message': 'Invalid To Address'
})
if value:
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_value',
'message': 'Invalid Value'
})
# check optional arguments
if nonce is None:
# check cache for nonce
nonce = await self.get_transaction_count(from_address)
else:
nonce = parse_int(nonce)
if nonce is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_nonce',
'message': 'Invalid Nonce'
})
if data is not None:
if isinstance(data, int):
data = hex(data)
if isinstance(data, str):
try:
data = data_decoder(data)
except binascii.Error:
pass
if not isinstance(data, bytes):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_data',
'message': 'Invalid Data field'
})
else:
data = b''
if gas is None:
try:
gas = await self.eth.eth_estimateGas(from_address,
to_address,
data=data,
value=value)
except JsonRPCError:
# this can occur if sending a transaction to a contract that doesn't match a valid method
# and the contract has no default method implemented
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_data',
'message': 'Invalid Data field'
})
else:
gas = parse_int(gas)
if gas is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_gas',
'message': 'Invalid Gas'
})
if gas_price is None:
gas_price = self.application.config['ethereum'].getint(
'default_gasprice', DEFAULT_GASPRICE)
else:
gas_price = parse_int(gas_price)
if gas_price is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_gas_price',
'message': 'Invalid Gas Price'
})
try:
tx = create_transaction(nonce=nonce,
gasprice=gas_price,
startgas=gas,
to=to_address,
value=value,
data=data,
network_id=self.network_id)
except InvalidTransaction as e:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': str(e)
})
if tx.intrinsic_gas_used > gas:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_transaction',
'message':
'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'
.format(tx.intrinsic_gas_used, gas)
})
transaction = encode_transaction(tx)
return transaction
async def create_transaction_skeleton(self, *, to_address, from_address, value=0, nonce=None, gas=None, gas_price=None, data=None):
if not validate_address(from_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address'})
if to_address is not None and not validate_address(to_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address'})
if from_address != from_address.lower() and not checksum_validate_address(from_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address Checksum'})
if to_address is not None and to_address != to_address.lower() and not checksum_validate_address(to_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address Checksum'})
if value:
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_value', 'message': 'Invalid Value'})
# check optional arguments
# check if we should ignore the given gasprice
# NOTE: only meant to be here while cryptokitty fever is pushing
# up gas prices... this shouldn't be perminant
# anytime the nonce is also set, use the provided gas (this is to
# support easier overwriting of transactions)
if gas_price is not None and nonce is None:
async with self.db:
whitelisted = await self.db.fetchrow("SELECT 1 FROM from_address_gas_price_whitelist WHERE address = $1", from_address)
if not whitelisted:
whitelisted = await self.db.fetchrow("SELECT 1 FROM to_address_gas_price_whitelist WHERE address = $1", to_address)
if not whitelisted:
gas_price = None
if nonce is None:
# check cache for nonce
nonce = await self.get_transaction_count(from_address)
else:
nonce = parse_int(nonce)
if nonce is None:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Invalid Nonce'})
if data is not None:
if isinstance(data, int):
data = hex(data)
if isinstance(data, str):
try:
data = data_decoder(data)
except binascii.Error:
pass
if not isinstance(data, bytes):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Invalid Data field'})
else:
data = b''
if gas is None:
try:
gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=value)
except JsonRPCError:
# this can occur if sending a transaction to a contract that doesn't match a valid method
# and the contract has no default method implemented
raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Unable to estimate gas for contract call'})
else:
gas = parse_int(gas)
if gas is None:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas', 'message': 'Invalid Gas'})
if gas_price is None:
# try and use cached gas station gas price
gas_station_gas_price = self.redis.get('gas_station_standard_gas_price')
if gas_station_gas_price:
gas_price = parse_int(gas_station_gas_price)
if gas_price is None:
gas_price = self.application.config['ethereum'].getint('default_gasprice', DEFAULT_GASPRICE)
else:
gas_price = parse_int(gas_price)
if gas_price is None:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas_price', 'message': 'Invalid Gas Price'})
try:
tx = create_transaction(nonce=nonce, gasprice=gas_price, startgas=gas,
to=to_address, value=value, data=data,
network_id=self.network_id)
except InvalidTransaction as e:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction', 'message': str(e)})
if tx.intrinsic_gas_used > gas:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'.format(
tx.intrinsic_gas_used, gas)})
transaction = encode_transaction(tx)
return {"tx": transaction, "gas": hex(gas), "gas_price": hex(gas_price), "nonce": hex(nonce), "value": hex(value)}
async def create_transaction_skeleton(self,
*,
to_address,
from_address,
value=0,
nonce=None,
gas=None,
gas_price=None,
data=None,
token_address=None):
if not validate_address(from_address):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_from_address',
'message': 'Invalid From Address'
})
if to_address is not None and not validate_address(to_address):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_to_address',
'message': 'Invalid To Address'
})
if from_address != from_address.lower(
) and not checksum_validate_address(from_address):
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_from_address',
'message': 'Invalid From Address Checksum'
})
if to_address is not None and to_address != to_address.lower(
) and not checksum_validate_address(to_address):
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_to_address',
'message': 'Invalid To Address Checksum'
})
# check if we should ignore the given gasprice
# NOTE: only meant to be here while cryptokitty fever is pushing
# up gas prices... this shouldn't be perminant
# anytime the nonce is also set, use the provided gas (this is to
# support easier overwriting of transactions)
if gas_price is not None and nonce is None:
async with self.db:
whitelisted = await self.db.fetchrow(
"SELECT 1 FROM from_address_gas_price_whitelist WHERE address = $1",
from_address)
if not whitelisted:
whitelisted = await self.db.fetchrow(
"SELECT 1 FROM to_address_gas_price_whitelist WHERE address = $1",
to_address)
if not whitelisted:
gas_price = None
if gas_price is None:
# try and use cached gas station gas price
gas_station_gas_price = await self.redis.get(
'gas_station_standard_gas_price')
if gas_station_gas_price:
gas_price = parse_int(gas_station_gas_price)
if gas_price is None:
gas_price = config['ethereum'].getint('default_gasprice',
DEFAULT_GASPRICE)
else:
gas_price = parse_int(gas_price)
if gas_price is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_gas_price',
'message': 'Invalid Gas Price'
})
if gas is not None:
gas = parse_int(gas)
if gas is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_gas',
'message': 'Invalid Gas'
})
if nonce is None:
# check cache for nonce
nonce = await self.get_transaction_count(from_address)
else:
nonce = parse_int(nonce)
if nonce is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_nonce',
'message': 'Invalid Nonce'
})
if data is not None:
if isinstance(data, int):
data = hex(data)
if isinstance(data, str):
try:
data = data_decoder(data)
except binascii.Error:
pass
if not isinstance(data, bytes):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_data',
'message': 'Invalid Data field'
})
else:
data = b''
# flag to force arguments into an erc20 token transfer
if token_address is not None:
if not validate_address(token_address):
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_token_address',
'message': 'Invalid Token Address'
})
if data != b'':
raise JsonRPCInvalidParamsError(
data={
'id': 'bad_arguments',
'message': 'Cannot include both data and token_address'
})
if isinstance(value, str) and value.lower() == "max":
# get the balance in the database
async with self.db:
value = await self.db.fetchval(
"SELECT value FROM token_balances "
"WHERE contract_address = $1 AND eth_address = $2",
token_address, from_address)
if value is None:
# get the value from the ethereum node
data = "0x70a08231000000000000000000000000" + from_address[
2:].lower()
value = await self.eth.eth_call(to_address=token_address,
data=data)
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_value',
'message': 'Invalid Value'
})
data = data_decoder(
"0xa9059cbb000000000000000000000000{}{:064x}".format(
to_address[2:].lower(), value))
token_value = value
value = 0
to_address = token_address
elif value:
if value == "max":
network_balance, balance, _, _ = await self.get_balances(
from_address)
if gas is None:
code = await self.eth.eth_getCode(to_address)
if code:
# we might have to do some work
try:
gas = await self.eth.eth_estimateGas(from_address,
to_address,
data=data,
value=0)
except JsonRPCError:
# no fallback function implemented in the contract means no ether can be sent to it
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_to_address',
'message':
'Cannot send payments to that address'
})
attempts = 0
# because the default function could do different things based on the eth sent, we make sure
# the value is suitable. if we get different values 3 times abort
while True:
if attempts > 2:
log.warning(
"Hit max attempts trying to get max value to send to contract '{}'"
.format(to_address))
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_to_address',
'message':
'Cannot send payments to that address'
})
value = balance - (gas_price * gas)
try:
gas_with_value = await self.eth.eth_estimateGas(
from_address,
to_address,
data=data,
value=value)
except JsonRPCError:
# no fallback function implemented in the contract means no ether can be sent to it
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_to_address',
'message':
'Cannot send payments to that address'
})
if gas_with_value != gas:
gas = gas_with_value
attempts += 1
continue
else:
break
else:
# normal address, 21000 gas per transaction
gas = 21000
value = balance - (gas_price * gas)
else:
# preset gas, run with it!
value = balance - (gas_price * gas)
else:
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_value',
'message': 'Invalid Value'
})
if gas is None:
try:
gas = await self.eth.eth_estimateGas(from_address,
to_address,
data=data,
value=value)
except JsonRPCError:
# this can occur if sending a transaction to a contract that doesn't match a valid method
# and the contract has no default method implemented.
# this can also happen if the current state of the blockchain means that submitting the
# transaction would fail (abort).
if token_address is not None:
# when dealing with erc20, this usually means the user's balance for that token isn't
# high enough, check that and throw an error if it's the case, and if not fall
# back to the standard invalid_data error
async with self.db:
bal = await self.db.fetchval(
"SELECT value FROM token_balances "
"WHERE contract_address = $1 AND eth_address = $2",
token_address, from_address)
if bal is not None:
bal = parse_int(bal)
if bal < token_value:
raise JsonRPCInsufficientFundsError(
data={
'id': 'insufficient_funds',
'message': 'Insufficient Funds'
})
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_data',
'message': 'Unable to estimate gas for contract call'
})
# if data is present, buffer gas estimate by 20%
if len(data) > 0:
gas = int(gas * 1.2)
try:
tx = create_transaction(nonce=nonce,
gasprice=gas_price,
startgas=gas,
to=to_address,
value=value,
data=data,
network_id=self.network_id)
except InvalidTransaction as e:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': str(e)
})
if tx.intrinsic_gas_used > gas:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_transaction',
'message':
'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'
.format(tx.intrinsic_gas_used, gas)
})
transaction = encode_transaction(tx)
return {
"tx": transaction,
"gas": hex(gas),
"gas_price": hex(gas_price),
"nonce": hex(nonce),
"value": hex(token_value) if token_address else hex(value)
}
async def _process_transaction_queue(self, ethereum_address):
log.debug("processing tx queue for {}".format(ethereum_address))
# check for un-scheduled transactions
async with self.db:
# get the last block number to use in ethereum calls
# to avoid race conditions in transactions being confirmed
# on the network before the block monitor sees and updates them in the database
last_blocknumber = (
await
self.db.fetchval("SELECT blocknumber FROM last_blocknumber"))
transactions_out = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'new' OR status = 'queued') "
"AND r IS NOT NULL "
# order by nonce reversed so that .pop() can
# be used in the loop below
"ORDER BY nonce DESC",
ethereum_address)
# any time the state of a transaction is changed we need to make
# sure those changes cascade down to the receiving address as well
# this keeps a list of all the receiving addresses that need to be
# checked after the current address's queue has been processed
addresses_to_check = set()
if transactions_out:
# TODO: make sure the block number isn't too far apart from the current
# if this is the case then we should just come back later!
# get the current network balance for this address
balance = await self.eth.eth_getBalance(ethereum_address,
block=last_blocknumber
or "latest")
# get the unconfirmed_txs
async with self.db:
unconfirmed_txs = await self.db.fetch(
"SELECT nonce, value, gas, gas_price FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'unconfirmed' "
"OR (status = 'confirmed' AND blocknumber > $2)) "
"ORDER BY nonce", ethereum_address, last_blocknumber or 0)
network_nonce = await self.eth.eth_getTransactionCount(
ethereum_address, block=last_blocknumber or "latest")
if unconfirmed_txs:
nonce = unconfirmed_txs[-1]['nonce'] + 1
balance -= sum(
parse_int(tx['value']) +
(parse_int(tx['gas']) * parse_int(tx['gas_price']))
for tx in unconfirmed_txs)
else:
# use the nonce from the network
nonce = network_nonce
# marker for whether a previous transaction had an error (signaling
# that all the following should also be an error
previous_error = False
# for each one, check if we can schedule them yet
while transactions_out:
transaction = transactions_out.pop()
# if there was a previous error in the queue, abort!
if previous_error:
log.info("Setting tx '{}' to error due to previous error".
format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# make sure the nonce is still valid
if nonce != transaction[
'nonce'] and network_nonce != transaction['nonce']:
# check if this is an overwrite
if transaction['status'] == 'new':
async with self.db:
old_tx = await self.db.fetchrow(
"SELECT * FROM transactions where from_address = $1 AND nonce = $2 AND hash != $3",
ethereum_address, transaction['nonce'],
transaction['hash'])
if old_tx:
if old_tx['status'] == 'error':
# expected state for overwrites
pass
elif old_tx['status'] == 'unconfirmed' or old_tx[
'status'] == 'confirmed':
previous_error = True
log.info((
"Setting tx '{}' to error due to another unconfirmed transaction"
"with nonce ({}) already existing in the system"
).format(transaction['hash'],
transaction['nonce']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(
transaction['to_address'])
continue
else:
# two transactions with the same nonce on the queue
# lets pick the one with the highest gas price and error the other
if transaction['nonce'] > old_tx['nonce']:
# lets use this one!
log.info((
"Setting tx '{}' to error due to another unconfirmed transaction"
"with nonce ({}) already existing in the system"
).format(old_tx['hash'],
transaction['nonce']))
await self.update_transaction(
old_tx['transaction_id'], 'error')
addresses_to_check.add(
old_tx['to_address'])
# make sure the other transaction is pulled out of the queue
try:
idx = next(i for i, e in enumerate(
transactions_out)
if e['transaction_id'] ==
old_tx['transaction_id'])
del transactions_out[idx]
except:
# old_tx not in the transactions_out list
pass
else:
# we'll use the other one
log.info((
"Setting tx '{}' to error due to another unconfirmed transaction"
"with nonce ({}) already existing in the system"
).format(old_tx['hash'],
transaction['nonce']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(
transaction['to_address'])
addresses_to_check.add(
transaction['from_address'])
# this case is actually pretty weird, so emptying the
# transactions_out so we restart the queue check
# completely
transactions_out = []
continue
else:
# well this is awkward! may as well let things go on in this case because
# it means a transaction in the nonce sequence is missing
pass
elif transaction['status'] == 'queued':
# then this and all the following transactions are now invalid
previous_error = True
log.info(
"Setting tx '{}' to error due to the nonce ({}) not matching the network ({})"
.format(transaction['hash'], transaction['nonce'],
nonce))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
else:
# this is a really weird state
# it's not clear what should be done here
log.error(
"Found unconfirmed transaction with out of order nonce for address: {}"
.format(ethereum_address))
return
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
cost = value + (gas * gas_price)
# check if the current balance is high enough to send to the network
if balance >= cost:
# check if gas price is high enough that it makes sense to send the transaction
safe_gas_price = parse_int(
await self.redis.get('gas_station_safelow_gas_price'))
if safe_gas_price and safe_gas_price > gas_price:
log.debug(
"Not queuing tx '{}' as current gas price would not support it"
.format(transaction['hash']))
# retry this address in a minute
manager_dispatcher.process_transaction_queue(
ethereum_address).delay(60)
# abort the rest of the processing after sending PNs for any "new" transactions
while transaction:
if transaction['status'] == 'new':
await self.update_transaction(
transaction['transaction_id'], 'queued')
transaction = transactions_out.pop(
) if transactions_out else None
break
# if so, send the transaction
# create the transaction
data = data_decoder(
transaction['data']) if transaction['data'] else b''
tx = create_transaction(nonce=transaction['nonce'],
value=value,
gasprice=gas_price,
startgas=gas,
to=transaction['to_address'],
data=data,
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
# make sure the signature was valid
if data_encoder(tx.sender) != ethereum_address:
# signature is invalid for the user
log.error(
"ERROR signature invalid for sender of tx: {}".
format(transaction['hash']))
log.error("queue: {}, db: {}, tx: {}".format(
ethereum_address, transaction['from_address'],
data_encoder(tx.sender)))
previous_error = True
addresses_to_check.add(transaction['to_address'])
await self.update_transaction(
transaction['transaction_id'], 'error')
continue
# send the transaction
try:
tx_encoded = encode_transaction(tx)
await self.eth.eth_sendRawTransaction(tx_encoded)
await self.update_transaction(
transaction['transaction_id'], 'unconfirmed')
except JsonRPCError as e:
# if something goes wrong with sending the transaction
# simply abort for now.
# TODO: depending on error, just break and queue to retry later
log.error(
"ERROR sending queued transaction: {}".format(
e.format()))
if e.message and (e.message.startswith(
"Transaction nonce is too low"
) or e.message.startswith(
"Transaction with the same hash was already imported"
)):
existing_tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
if existing_tx:
if existing_tx['blockNumber']:
await self.update_transaction(
transaction['transaction_id'],
'confirmed')
else:
await self.update_transaction(
transaction['transaction_id'],
'unconfirmed')
continue
previous_error = True
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# adjust the balance values for checking the other transactions
balance -= cost
if nonce == transaction['nonce']:
nonce += 1
continue
else:
# make sure the pending_balance would support this transaction
# otherwise there's no way this transaction will be able to
# be send, so trigger a failure on all the remaining transactions
async with self.db:
transactions_in = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE to_address = $1 "
"AND ("
"(status = 'new' OR status = 'queued' OR status = 'unconfirmed') "
"OR (status = 'confirmed' AND blocknumber > $2))",
ethereum_address, last_blocknumber or 0)
# TODO: test if loops in the queue chain are problematic
pending_received = sum(
(parse_int(p['value']) or 0) for p in transactions_in)
if balance + pending_received < cost:
previous_error = True
log.info(
"Setting tx '{}' to error due to insufficient pending balance"
.format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
else:
if any(t['blocknumber'] is not None
and t['blocknumber'] > last_blocknumber
for t in transactions_in):
addresses_to_check.add(ethereum_address)
# there's no reason to continue on here since all the
# following transaction in the queue cannot be processed
# until this one is
# but we still need to send PNs for any "new" transactions
while transaction:
if transaction['status'] == 'new':
await self.update_transaction(
transaction['transaction_id'], 'queued')
transaction = transactions_out.pop(
) if transactions_out else None
break
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
manager_dispatcher.process_transaction_queue(address)
if transactions_out:
manager_dispatcher.process_transaction_queue(ethereum_address)
async def sanity_check(self, frequency):
async with self.db:
rows = await self.db.fetch(
"SELECT DISTINCT from_address FROM transactions WHERE (status = 'unconfirmed' OR status = 'queued' OR status = 'new') "
"AND v IS NOT NULL AND created < (now() AT TIME ZONE 'utc') - interval '3 minutes'"
)
rows2 = await self.db.fetch(
"WITH t1 AS (SELECT DISTINCT from_address FROM transactions WHERE (status = 'new' OR status = 'queued') AND v IS NOT NULL), "
"t2 AS (SELECT from_address, COUNT(*) FROM transactions WHERE (status = 'unconfirmed' AND v IS NOT NULL) GROUP BY from_address) "
"SELECT t1.from_address FROM t1 LEFT JOIN t2 ON t1.from_address = t2.from_address WHERE t2.count IS NULL;"
)
if rows or rows2:
log.debug(
"sanity check found {} addresses with potential problematic transactions"
.format(len(rows) + len(rows2)))
rows = set([row['from_address'] for row in rows
]).union(set([row['from_address'] for row in rows2]))
addresses_to_check = set()
old_and_unconfirmed = []
for ethereum_address in rows:
# check on queued transactions
async with self.db:
queued_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'new' OR status = 'queued') AND v IS NOT NULL",
ethereum_address)
if queued_transactions:
# make sure there are pending incoming transactions
async with self.db:
incoming_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE to_address = $1 "
"AND (status = 'unconfirmed' OR status = 'queued' OR status = 'new')",
ethereum_address)
if not incoming_transactions:
log.error(
"ERROR: {} has transactions in it's queue, but no unconfirmed transactions!"
.format(ethereum_address))
# trigger queue processing as last resort
addresses_to_check.add(ethereum_address)
else:
# check health of the incoming transaction
for transaction in incoming_transactions:
if transaction['v'] is None:
try:
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
except:
log.exception(
"Error getting transaction {} in sanity check",
transaction['hash'])
continue
if tx is None:
log.warning(
"external transaction (id: {}) no longer found on nodes"
.format(transaction['transaction_id']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(ethereum_address)
elif tx['blockNumber'] is not None:
log.warning(
"external transaction (id: {}) confirmed on node, but wasn't confirmed in db"
.format(transaction['transaction_id']))
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(ethereum_address)
# no need to continue with dealing with unconfirmed transactions if there are queued ones
continue
async with self.db:
unconfirmed_transactions = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND status = 'unconfirmed' AND v IS NOT NULL",
ethereum_address)
if unconfirmed_transactions:
for transaction in unconfirmed_transactions:
# check on unconfirmed transactions first
if transaction['status'] == 'unconfirmed':
# we neehed to check the true status of unconfirmed transactions
# as the block monitor may be inbetween calls and not have seen
# this transaction to mark it as confirmed.
try:
tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
except:
log.exception(
"Error getting transaction {} in sanity check",
transaction['hash'])
continue
# sanity check to make sure the tx still exists
if tx is None:
# if not, try resubmit
# NOTE: it may just be an issue with load balanced nodes not seeing all pending transactions
# so we don't want to adjust the status of the transaction at all at this stage
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
data = data_decoder(
transaction['data']
) if transaction['data'] else b''
tx = create_transaction(
nonce=transaction['nonce'],
value=value,
gasprice=gas_price,
startgas=gas,
to=transaction['to_address'],
data=data,
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
if calculate_transaction_hash(
tx) != transaction['hash']:
log.warning(
"error resubmitting transaction {}: regenerating tx resulted in a different hash"
.format(transaction['hash']))
else:
tx_encoded = encode_transaction(tx)
try:
await self.eth.eth_sendRawTransaction(
tx_encoded)
addresses_to_check.add(
transaction['from_address'])
except Exception as e:
# note: usually not critical, don't panic
log.warning(
"error resubmitting transaction {}: {}"
.format(transaction['hash'], str(e)))
elif tx['blockNumber'] is not None:
# confirmed! update the status
await self.update_transaction(
transaction['transaction_id'], 'confirmed')
addresses_to_check.add(transaction['from_address'])
addresses_to_check.add(transaction['to_address'])
else:
old_and_unconfirmed.append(transaction['hash'])
if len(old_and_unconfirmed):
log.warning(
"WARNING: {} transactions are old and unconfirmed!".format(
len(old_and_unconfirmed)))
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
manager_dispatcher.process_transaction_queue(address)
if frequency:
manager_dispatcher.sanity_check(frequency).delay(frequency)
async def _process_transaction_queue(self, ethereum_address):
log.info("processing tx queue for {}".format(ethereum_address))
# check for un-scheduled transactions
async with self.db:
# get the last block number to use in ethereum calls
# to avoid race conditions in transactions being confirmed
# on the network before the block monitor sees and updates them in the database
last_blocknumber = (
await
self.db.fetchval("SELECT blocknumber FROM last_blocknumber"))
transactions_out = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND (status is NULL OR status = 'queued') "
"AND r IS NOT NULL "
# order by nonce reversed so that .pop() can
# be used in the loop below
"ORDER BY nonce DESC",
ethereum_address)
# any time the state of a transaction is changed we need to make
# sure those changes cascade down to the receiving address as well
# this keeps a list of all the receiving addresses that need to be
# checked after the current address's queue has been processed
addresses_to_check = set()
if transactions_out:
# TODO: make sure the block number isn't too far apart from the current
# if this is the case then we should just come back later!
# get the current network balance for this address
balance = await self.eth.eth_getBalance(ethereum_address,
block=last_blocknumber
or "latest")
# get the unconfirmed_txs
async with self.db:
unconfirmed_txs = await self.db.fetch(
"SELECT nonce, value, gas, gas_price FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'unconfirmed' "
"OR (status = 'confirmed' AND blocknumber > $2)) "
"ORDER BY nonce", ethereum_address, last_blocknumber or 0)
if unconfirmed_txs:
nonce = unconfirmed_txs[-1]['nonce'] + 1
balance -= sum(
parse_int(tx['value']) +
(parse_int(tx['gas']) * parse_int(tx['gas_price']))
for tx in unconfirmed_txs)
else:
# use the nonce from the network
nonce = await self.eth.eth_getTransactionCount(
ethereum_address, block=last_blocknumber or "latest")
# marker for whether a previous transaction had an error (signaling
# that all the following should also be an error
previous_error = False
# for each one, check if we can schedule them yet
while transactions_out:
transaction = transactions_out.pop()
# if there was a previous error in the queue, abort!
if previous_error:
log.info("Setting tx '{}' to error due to previous error".
format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# make sure the nonce is still valid
if nonce != transaction['nonce']:
# then this and all the following transactions are now invalid
previous_error = True
log.info(
"Setting tx '{}' to error due to the nonce ({}) not matching the network ({})"
.format(transaction['hash'], transaction['nonce'],
nonce))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
cost = value + (gas * gas_price)
# check if the current balance is high enough to send to the network
if balance >= cost:
# if so, send the transaction
# create the transaction
data = data_decoder(
transaction['data']) if transaction['data'] else b''
tx = create_transaction(nonce=nonce,
value=value,
gasprice=gas_price,
startgas=gas,
to=transaction['to_address'],
data=data,
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
# make sure the signature was valid
if data_encoder(tx.sender) != ethereum_address:
# signature is invalid for the user
log.error(
"ERROR signature invalid for sender of tx: {}".
format(transaction['hash']))
log.error("queue: {}, db: {}, tx: {}".format(
ethereum_address, transaction['from_address'],
data_encoder(tx.sender)))
previous_error = True
addresses_to_check.add(transaction['to_address'])
await self.update_transaction(
transaction['transaction_id'], 'error')
continue
# send the transaction
try:
tx_encoded = encode_transaction(tx)
await self.eth.eth_sendRawTransaction(tx_encoded)
await self.update_transaction(
transaction['transaction_id'], 'unconfirmed')
except JsonRPCError as e:
# if something goes wrong with sending the transaction
# simply abort for now.
# TODO: depending on error, just break and queue to retry later
log.error(
"ERROR sending queued transaction: {}".format(
e.format()))
previous_error = True
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# adjust the balance values for checking the other transactions
balance -= cost
nonce += 1
continue
else:
# make sure the pending_balance would support this transaction
# otherwise there's no way this transaction will be able to
# be send, so trigger a failure on all the remaining transactions
async with self.db:
transactions_in = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE to_address = $1 "
"AND ("
"(status is NULL OR status = 'queued' OR status = 'unconfirmed') "
"OR (status = 'confirmed' AND blocknumber > $2))",
ethereum_address, last_blocknumber or 0)
# TODO: test if loops in the queue chain are problematic
pending_received = sum(
(parse_int(p['value']) or 0) for p in transactions_in)
if balance + pending_received < cost:
previous_error = True
log.info(
"Setting tx '{}' to error due to insufficient pending balance"
.format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
else:
if any(t['blocknumber'] is not None
and t['blocknumber'] > last_blocknumber
for t in transactions_in):
addresses_to_check.add(ethereum_address)
# there's no reason to continue on here since all the
# following transaction in the queue cannot be processed
# until this one is
# but we still need to send PNs for any "new" transactions
while transaction:
if transaction['status'] is None:
await self.update_transaction(
transaction['transaction_id'], 'queued')
transaction = transactions_out.pop(
) if transactions_out else None
break
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
self.tasks.process_transaction_queue(address)
if transactions_out:
self.tasks.process_transaction_queue(ethereum_address)
def do_POST(self):
# TODO: figure out why read is blocking here
data = self.rfile.read(len(self.rfile.peek()))
data = data.decode('utf-8')
data = json.loads(data)
if self.path == "/v1/tx/skel":
gas_price = parse_int(
data['gas_price']) if 'gas_price' in data else DEFAULT_GASPRICE
gas = parse_int(data['gas']) if 'gas' in data else DEFAULT_STARTGAS
nonce = parse_int(data['nonce']) if 'nonce' in data else 0
if 'value' not in data or 'from' not in data or 'to' not in data:
self.write_data(
400, {
'errors': [{
'id': 'bad_arguments',
'message': 'Bad Arguments'
}]
})
return
value = parse_int(data['value'])
to_address = data['to']
from_address = data['from']
if not validate_address(to_address):
self.write_data(
400, {
'errors': [{
'id': 'invalid_to_address',
'message': 'Invalid To Address'
}]
})
return
if not validate_address(from_address):
self.write_data(
400, {
'errors': [{
'id': 'invalid_from_address',
'message': 'Invalid From Address'
}]
})
return
tx = create_transaction(nonce=nonce,
gasprice=gas_price,
startgas=gas,
to=to_address,
value=value)
transaction = encode_transaction(tx)
self.write_data(
200, {
"tx_data": {
"nonce": hex(nonce),
"from": from_address,
"to": to_address,
"value": hex(value),
"startGas": hex(gas),
"gasPrice": hex(gas_price)
},
"tx": transaction
})
elif self.path == "/v1/tx":
tx = decode_transaction(data['tx'])
if 'signature' in data:
sig = data_decoder(data['signature'])
add_signature_to_transaction(tx, sig)
self.write_data(200, {"tx_hash": data_encoder(tx.hash)})
else:
self.write_data(404)
