async def test_only_apps_query(self):
data_encoder, private_key_to_address
users = [('bob{}'.format(i),
private_key_to_address(data_encoder(os.urandom(32))), False)
for i in range(6)]
bots = [('bot{}'.format(i),
private_key_to_address(data_encoder(os.urandom(32))), True)
for i in range(4)]
async with self.pool.acquire() as con:
for args in users + bots:
await con.execute(
"INSERT INTO users (username, toshi_id, is_app) VALUES ($1, $2, $3)",
*args)
resp = await self.fetch("/search/user?query=bo", method="GET")
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
self.assertEqual(len(body['results']), 10)
resp = await self.fetch("/search/user?query=bo&apps=false",
method="GET")
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
self.assertEqual(len(body['results']), 6)
resp = await self.fetch("/search/user?query=bo&apps=true",
method="GET")
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
self.assertEqual(len(body['results']), 4)
def test_add_signature_to_transaction_with_netowrk_id(self):
for network_id in [1, 2, 66, 100]:
sender_private_key = "0x0164f7c7399f4bb1eafeaae699ebbb12050bc6a50b2836b9ca766068a9d000c0"
sender_address = "0xde3d2d9dd52ea80f7799ef4791063a5458d13913"
to_address = "0x056db290f8ba3250ca64a45d16284d04bc6f5fbf"
value = 10000000000
nonce = 1048576
data = b''
gasprice = DEFAULT_GASPRICE
startgas = DEFAULT_STARTGAS
network_id = 1
tx1 = Transaction(nonce, gasprice, startgas, to_address, value,
data, network_id, 0, 0)
tx = encode_transaction(tx1)
tx1.sign(data_decoder(sender_private_key), network_id=network_id)
expected_signed_tx = encode_transaction(tx1)
sig = data_encoder(signature_from_transaction(tx1))
signed_tx = add_signature_to_transaction(tx, sig)
self.assertEqual(signed_tx, expected_signed_tx)
tx_obj = decode_transaction(tx)
add_signature_to_transaction(tx_obj, sig)
self.assertEqual(tx_obj.network_id, network_id)
self.assertEqual(data_encoder(tx_obj.sender), sender_address)
self.assertEqual(encode_transaction(tx_obj), expected_signed_tx)
async def faucet(self,
to,
value,
*,
from_private_key=FAUCET_PRIVATE_KEY,
startgas=None,
gasprice=DEFAULT_GASPRICE,
nonce=None,
data=b"",
wait_on_confirmation=True):
if isinstance(from_private_key, str):
from_private_key = data_decoder(from_private_key)
from_address = private_key_to_address(from_private_key)
ethclient = JsonRPCClient(config['ethereum']['url'])
to = data_decoder(to)
if len(to) not in (20, 0):
raise Exception(
'Addresses must be 20 or 0 bytes long (len was {})'.format(
len(to)))
if nonce is None:
nonce = await ethclient.eth_getTransactionCount(from_address)
balance = await ethclient.eth_getBalance(from_address)
if startgas is None:
startgas = await ethclient.eth_estimateGas(from_address,
to,
data=data,
nonce=nonce,
value=value,
gasprice=gasprice)
tx = Transaction(nonce, gasprice, startgas, to, value, data, 0, 0, 0)
if balance < (tx.value + (tx.startgas * tx.gasprice)):
raise Exception("Faucet doesn't have enough funds")
tx.sign(from_private_key)
tx_encoded = data_encoder(rlp.encode(tx, Transaction))
tx_hash = await ethclient.eth_sendRawTransaction(tx_encoded)
while wait_on_confirmation:
resp = await ethclient.eth_getTransactionByHash(tx_hash)
if resp is None or resp['blockNumber'] is None:
await asyncio.sleep(0.1)
else:
break
if to == b'':
print("contract address: {}".format(data_encoder(tx.creates)))
return tx_hash
async def deploy_contract(self,
bytecode,
*,
from_private_key=FAUCET_PRIVATE_KEY,
startgas=None,
gasprice=DEFAULT_GASPRICE,
wait_on_confirmation=True):
if isinstance(from_private_key, str):
from_private_key = data_decoder(from_private_key)
from_address = private_key_to_address(from_private_key)
ethclient = JsonRPCClient(config['ethereum']['url'])
nonce = await ethclient.eth_getTransactionCount(from_address)
balance = await ethclient.eth_getBalance(from_address)
gasestimate = await ethclient.eth_estimateGas(from_address,
'',
data=bytecode,
nonce=nonce,
value=0,
gasprice=gasprice)
if startgas is None:
startgas = gasestimate
elif gasestimate > startgas:
raise Exception(
"Estimated gas usage is larger than the provided gas")
tx = Transaction(nonce, gasprice, startgas, '', 0, bytecode, 0, 0, 0)
if balance < (tx.value + (tx.startgas * tx.gasprice)):
raise Exception("Faucet doesn't have enough funds")
tx.sign(from_private_key)
tx_encoded = data_encoder(rlp.encode(tx, Transaction))
tx_hash = await ethclient.eth_sendRawTransaction(tx_encoded)
contract_address = data_encoder(tx.creates)
while wait_on_confirmation:
resp = await ethclient.eth_getTransactionByHash(tx_hash)
if resp is None or resp['blockNumber'] is None:
await asyncio.sleep(0.1)
else:
code = await ethclient.eth_getCode(contract_address)
if code == '0x':
raise Exception("Failed to deploy contract")
break
return tx_hash, contract_address
async def store_tx(status, created, updated=None):
tx_hash = data_encoder(os.urandom(64))
from_addr = data_encoder(os.urandom(20))
value = random.randint(10**15, 10**20)
if updated is None:
updated = created
async with self.pool.acquire() as con:
await con.execute(
"INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, status, created, updated) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
tx_hash, from_addr, addr, 1, hex(value),
hex(DEFAULT_STARTGAS), hex(DEFAULT_GASPRICE), status,
datetime.utcfromtimestamp(created),
datetime.utcfromtimestamp(updated))
async def test_cancel_transaction(self):
listener = MockTaskListener(self._app)
listener.start_task_listener()
tx = create_transaction(nonce=0, gasprice=10 ** 10, startgas=21000, to=TEST_ADDRESS, value=10 ** 18)
tx = sign_transaction(tx, FAUCET_PRIVATE_KEY)
tx_hash = calculate_transaction_hash(tx)
from_address = FAUCET_ADDRESS
to_address = TEST_ADDRESS
async with self.pool.acquire() as con:
await con.execute("INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, data, v, r, s, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
tx_hash, from_address, to_address, tx.nonce,
hex(tx.value), hex(tx.startgas), hex(tx.gasprice),
data_encoder(tx.data), hex(tx.v), hex(tx.r), hex(tx.s),
'queued')
signature = personal_sign(FAUCET_PRIVATE_KEY, "Cancel transaction " + tx_hash)
resp = await self.fetch("/tx/cancel", method="POST", body={"tx_hash": tx_hash, "signature": signature})
self.assertResponseCodeEqual(resp, 204)
tx_id, status = await listener.get()
self.assertEqual(tx_id, 1)
self.assertEqual(status, 'error')
await listener.stop_task_listener()
async def test_create_and_send_transaction_with_separate_sig(self):
body = {"from": FAUCET_ADDRESS, "to": TEST_ADDRESS, "value": 10**10}
resp = await self.fetch("/tx/skel", method="POST", body=body)
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx = decode_transaction(body['tx'])
tx = sign_transaction(tx, FAUCET_PRIVATE_KEY)
sig = signature_from_transaction(tx)
body = {"tx": body['tx'], "signature": data_encoder(sig)}
resp = await self.fetch("/tx", method="POST", body=body)
self.assertEqual(resp.code, 200, resp.body)
# ensure we get a tracking events
self.assertEqual((await self.next_tracking_event())[0], None)
body = json_decode(resp.body)
tx_hash = body['tx_hash']
async def check_db():
async with self.pool.acquire() as con:
rows = await con.fetch(
"SELECT * FROM transactions WHERE nonce = $1", tx.nonce)
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0]['hash'], tx_hash)
await self.wait_on_tx_confirmation(tx_hash)
async def test_large_volume_of_txs(self):
"""Tests that the service can handle a large volume of transactions
from different users
"""
num_of_txs = 10
# create sending user keys
from_keys = [os.urandom(32) for _ in range(num_of_txs)]
# create receiver user addresses
to_addresses = [data_encoder(os.urandom(20)) for _ in range(num_of_txs)]
# faucet sending users
txs = []
for key in from_keys:
addr = private_key_to_address(key)
txs.append((await self.send_tx(FAUCET_PRIVATE_KEY, addr, 10 * (10 ** 18))))
# send transactions
for key, address in zip(from_keys, to_addresses):
txs.append((await self.send_tx(key, address, 10 ** 18)))
while True:
async with self.pool.acquire() as con:
rows = await con.fetch("SELECT status, COUNT(*) FROM transactions WHERE hash = ANY($1) GROUP BY status",
txs)
s = []
for row in rows:
s.append("{}: {}".format(row['status'], row['count']))
if row['status'] == 'confirmed' and row['count'] == len(txs):
return
if row['status'] == 'queued' and row['count'] == 1:
async with self.pool.acquire() as con:
row = await con.fetchrow("SELECT * FROM transactions WHERE status = 'queued'")
print(row)
print(', '.join(s))
await asyncio.sleep(1)
async def test_cancel_transaction_fails_if_not_queued(self):
tx = create_transaction(nonce=0, gasprice=10 ** 10, startgas=21000, to=TEST_ADDRESS, value=10 ** 18)
tx = sign_transaction(tx, FAUCET_PRIVATE_KEY)
tx_hash = calculate_transaction_hash(tx)
from_address = FAUCET_ADDRESS
to_address = TEST_ADDRESS
async with self.pool.acquire() as con:
await con.execute("INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, data, v, r, s, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
tx_hash, from_address, to_address, tx.nonce,
hex(tx.value), hex(tx.startgas), hex(tx.gasprice),
data_encoder(tx.data), hex(tx.v), hex(tx.r), hex(tx.s),
'unconfirmed')
signature = personal_sign(FAUCET_PRIVATE_KEY, "Cancel transaction " + tx_hash)
resp = await self.fetch("/tx/cancel", method="POST", body={"tx_hash": tx_hash, "signature": signature})
self.assertResponseCodeEqual(resp, 400)
async with self.pool.acquire() as con:
await con.execute("UPDATE transactions SET status = 'confirmed'")
resp = await self.fetch("/tx/cancel", method="POST", body={"tx_hash": tx_hash, "signature": signature})
self.assertResponseCodeEqual(resp, 400)
async with self.pool.acquire() as con:
await con.execute("UPDATE transactions SET status = 'error'")
resp = await self.fetch("/tx/cancel", method="POST", body={"tx_hash": tx_hash, "signature": signature})
self.assertResponseCodeEqual(resp, 400)
async def test_sending_transactions_and_storing_the_hash_correctly(
self, *, push_client):
"""This test is born out of the fact that `UnsingedTransaction.hash` always
calculates the hash of the transaction without the signature, even after `.sign`
has been called on the transaction. This caused errors in what was being stored
in the database and incorrectly detecting transaction overwrites.
This test exposed the behaviour correctly so that it could be fixed"""
# register for GCM PNs
body = {"registration_id": TEST_GCM_ID, "address": TEST_ID_ADDRESS}
resp = await self.fetch_signed("/gcm/register",
signing_key=TEST_ID_KEY,
method="POST",
body=body)
self.assertResponseCodeEqual(resp, 204, resp.body)
body = {"from": FAUCET_ADDRESS, "to": TEST_ID_ADDRESS, "value": 10**10}
resp = await self.fetch("/tx/skel", method="POST", body=body)
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx = decode_transaction(body['tx'])
tx = sign_transaction(tx, FAUCET_PRIVATE_KEY)
sig = signature_from_transaction(tx)
body = {"tx": body['tx'], "signature": data_encoder(sig)}
resp = await self.fetch("/tx", method="POST", body=body)
self.assertEqual(resp.code, 200, resp.body)
body = json_decode(resp.body)
tx_hash = body['tx_hash']
async def check_db():
async with self.pool.acquire() as con:
rows = await con.fetch(
"SELECT * FROM transactions WHERE nonce = $1", tx.nonce)
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0]['hash'], tx_hash)
if rows[0]['status'] is not None:
self.assertEqual(rows[0]['status'], 'unconfirmed')
self.assertIsNone(rows[0]['error'])
await self.wait_on_tx_confirmation(tx_hash, check_db)
while True:
token, payload = await push_client.get()
message = parse_sofa_message(payload['message'])
self.assertIsInstance(message, SofaPayment)
self.assertEqual(message['txHash'], tx_hash)
if message['status'] == "confirmed":
break
async def on_login(self, address):
num = int(data_encoder(os.urandom(16))[2:], 16)
token = b62encode(num)
await self.redis.set('{}{}'.format(AUTH_TOKEN_REDIS_PREFIX, token),
address,
expire=AUTH_TOKEN_EXIPRY)
self.write({'auth_token': token})
async def test_create_and_send_transaction(self):
val = 10 ** 10
body = {
"from": FAUCET_ADDRESS,
"to": TEST_ADDRESS,
"value": val
}
resp = await self.fetch("/tx/skel", method="POST", body=body)
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx = sign_transaction(body['tx'], FAUCET_PRIVATE_KEY)
body = {
"tx": tx
}
resp = await self.fetch("/tx", method="POST", body=body)
self.assertEqual(resp.code, 200, resp.body)
# ensure we get a tracking events
self.assertEqual((await self.next_tracking_event())[0], None)
body = json_decode(resp.body)
tx_hash = body['tx_hash']
tx = decode_transaction(tx)
self.assertEqual(tx_hash, data_encoder(tx.hash))
async with self.pool.acquire() as con:
rows = await con.fetch("SELECT * FROM transactions WHERE nonce = $1", tx.nonce)
self.assertEqual(len(rows), 1)
# wait for a push notification
await self.wait_on_tx_confirmation(tx_hash)
while True:
async with self.pool.acquire() as con:
row = await con.fetchrow("SELECT * FROM transactions WHERE nonce = $1", tx.nonce)
if row['status'] == 'confirmed':
break
# make sure updated field is updated
self.assertGreater(row['updated'], row['created'])
# make sure balance is returned correctly
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), val)
self.assertEqual(parse_int(data['unconfirmed_balance']), val)
async def test_send_txs_for_all_networks(self):
tx = create_transaction(nonce=1048576, gasprice=20 * 10**9, startgas=21000,
to="0x3535353535353535353535353535353535353535",
value=10**18, data=b'')
sign_transaction(tx, FAUCET_PRIVATE_KEY)
resp = await self.fetch("/tx", method="POST", body={
"tx": encode_transaction(tx)
})
self.assertEqual(resp.code, 200, resp.body)
await self.wait_on_tx_confirmation(data_encoder(tx.hash))
async def test_create_tx_with_no_to_address(self):
body = {
"from": FAUCET_ADDRESS,
"data": data_encoder(b"Hello World"),
"value": 0,
"gas": 530000,
}
resp = await self.fetch("/tx/skel", method="POST", body=body)
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx = sign_transaction(body['tx'], FAUCET_PRIVATE_KEY)
body = {
"tx": tx
}
resp = await self.fetch("/tx", method="POST", body=body)
self.assertEqual(resp.code, 200, resp.body)
body = json_decode(resp.body)
tx_hash = body['tx_hash']
tx = decode_transaction(tx)
self.assertEqual(tx_hash, data_encoder(tx.hash))
async with self.pool.acquire() as con:
rows = await con.fetch("SELECT * FROM transactions WHERE nonce = $1", tx.nonce)
self.assertEqual(len(rows), 1)
print(rows[0])
await self.wait_on_tx_confirmation(tx_hash)
def test_encode_decode_transaction(self):
sender_private_key = "0x0164f7c7399f4bb1eafeaae699ebbb12050bc6a50b2836b9ca766068a9d000c0"
sender_address = "0xde3d2d9dd52ea80f7799ef4791063a5458d13913"
to_address = "0x056db290f8ba3250ca64a45d16284d04bc6f5fbf"
value = 10000000000
nonce = 1048576
data = b''
gasprice = 20000000000
startgas = DEFAULT_STARTGAS
expected_tx_hash = "0x2f321aa116146a9bc62b61c76508295f708f42d56340c9e613ebfc27e33f240c"
tx1 = Transaction(nonce, gasprice, startgas, to_address, value, data)
tx1.sign(data_decoder(sender_private_key))
self.assertEqual(data_encoder(tx1.hash), expected_tx_hash)
# rlputx1 = rlp.encode(tx1, UnsignedTransaction)
# rlpstx1 = rlp.encode(tx1, Transaction)
tx1 = Transaction(nonce, gasprice, startgas, to_address, value, data)
enc1 = rlp.encode(tx1, UnsignedTransaction)
tx2 = rlp.decode(enc1, UnsignedTransaction)
tx2.sign(data_decoder(sender_private_key))
tx3 = Transaction(tx2.nonce, tx2.gasprice, tx2.startgas, tx2.to,
tx2.value, tx2.data)
tx3.sign(data_decoder(sender_private_key))
self.assertEqual(data_encoder(tx3.sender), sender_address)
self.assertEqual(data_encoder(tx3.hash), expected_tx_hash)
self.assertEqual(data_encoder(tx2.sender), sender_address)
# NOTE: this is false because tx2 still thinks it's an unsigned tx
# so it doesn't include the signature variables in the tx
# if this suddenly starts failing, it means the behaviour
# has been modified in the library
self.assertNotEqual(data_encoder(tx2.hash), expected_tx_hash)
async def on_login(self, address):
num = int(data_encoder(os.urandom(16))[2:], 16)
token = b62encode(num)
async with self.db:
row = await self.db.fetchrow(
"SELECT * FROM users where toshi_id = $1", address)
if row is None:
raise JSONHTTPError(401)
await self.db.execute(
"INSERT INTO auth_tokens (token, address) VALUES ($1, $2)",
token, address)
await self.db.commit()
self.write({'auth_token': token})
async def test_contact_list_query(self):
total_users = 10000
# NOTE: this seems to be the limit for query string length
# that the server will accept
query_users = 1255
users = [(data_encoder(os.urandom(20)), hex(i))
for i in range(total_users)]
bad_users = [
"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"0xcccccccccccccccccccccccccccccccccccccccc"
]
bad_users.extend(user[0] for user in users[:3])
async with self.pool.acquire() as con:
await con.executemany(
"INSERT INTO users (toshi_id, username, active) VALUES ($1, $2, false)",
users)
resp = await self.fetch("/search/user?{}".format("&".join(
"toshi_id={}".format(toshi_id)
for toshi_id, _ in users[:query_users])))
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
self.assertIn('results', body)
self.assertEqual(len(body['results']), query_users)
for u, r in zip(users, body['results']):
self.assertEqual(u[0], r['toshi_id'])
resp = await self.fetch("/search/user?{}".format("&".join(
"toshi_id={}".format(toshi_id) for toshi_id in bad_users)))
body = json_decode(resp.body)
self.assertIn('results', body)
self.assertEqual(len(body['results']), 3)
# make sure we're safe from injection
inject = "0')) AS a (id) ON u.toshi_id = a.id; DELETE FROM users; SELECT u.* FROM users u JOIN ( VALUES ('0x0000000000000000000000000000000000000000"
resp = await self.fetch("/search/user?toshi_id={}".format(
quote_arg(inject)))
self.assertEqual(resp.code, 400)
async def _process_transaction_queue(self, ethereum_address):
log.debug("processing tx queue for {}".format(ethereum_address))
# check for un-scheduled transactions
async with self.db:
# get the last block number to use in ethereum calls
# to avoid race conditions in transactions being confirmed
# on the network before the block monitor sees and updates them in the database
last_blocknumber = (
await
self.db.fetchval("SELECT blocknumber FROM last_blocknumber"))
transactions_out = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'new' OR status = 'queued') "
"AND r IS NOT NULL "
# order by nonce reversed so that .pop() can
# be used in the loop below
"ORDER BY nonce DESC",
ethereum_address)
# any time the state of a transaction is changed we need to make
# sure those changes cascade down to the receiving address as well
# this keeps a list of all the receiving addresses that need to be
# checked after the current address's queue has been processed
addresses_to_check = set()
if transactions_out:
# TODO: make sure the block number isn't too far apart from the current
# if this is the case then we should just come back later!
# get the current network balance for this address
balance = await self.eth.eth_getBalance(ethereum_address,
block=last_blocknumber
or "latest")
# get the unconfirmed_txs
async with self.db:
unconfirmed_txs = await self.db.fetch(
"SELECT nonce, value, gas, gas_price FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'unconfirmed' "
"OR (status = 'confirmed' AND blocknumber > $2)) "
"ORDER BY nonce", ethereum_address, last_blocknumber or 0)
network_nonce = await self.eth.eth_getTransactionCount(
ethereum_address, block=last_blocknumber or "latest")
if unconfirmed_txs:
nonce = unconfirmed_txs[-1]['nonce'] + 1
balance -= sum(
parse_int(tx['value']) +
(parse_int(tx['gas']) * parse_int(tx['gas_price']))
for tx in unconfirmed_txs)
else:
# use the nonce from the network
nonce = network_nonce
# marker for whether a previous transaction had an error (signaling
# that all the following should also be an error
previous_error = False
# for each one, check if we can schedule them yet
while transactions_out:
transaction = transactions_out.pop()
# if there was a previous error in the queue, abort!
if previous_error:
log.info("Setting tx '{}' to error due to previous error".
format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# make sure the nonce is still valid
if nonce != transaction[
'nonce'] and network_nonce != transaction['nonce']:
# check if this is an overwrite
if transaction['status'] == 'new':
async with self.db:
old_tx = await self.db.fetchrow(
"SELECT * FROM transactions where from_address = $1 AND nonce = $2 AND hash != $3",
ethereum_address, transaction['nonce'],
transaction['hash'])
if old_tx:
if old_tx['status'] == 'error':
# expected state for overwrites
pass
elif old_tx['status'] == 'unconfirmed' or old_tx[
'status'] == 'confirmed':
previous_error = True
log.info((
"Setting tx '{}' to error due to another unconfirmed transaction"
"with nonce ({}) already existing in the system"
).format(transaction['hash'],
transaction['nonce']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(
transaction['to_address'])
continue
else:
# two transactions with the same nonce on the queue
# lets pick the one with the highest gas price and error the other
if transaction['nonce'] > old_tx['nonce']:
# lets use this one!
log.info((
"Setting tx '{}' to error due to another unconfirmed transaction"
"with nonce ({}) already existing in the system"
).format(old_tx['hash'],
transaction['nonce']))
await self.update_transaction(
old_tx['transaction_id'], 'error')
addresses_to_check.add(
old_tx['to_address'])
# make sure the other transaction is pulled out of the queue
try:
idx = next(i for i, e in enumerate(
transactions_out)
if e['transaction_id'] ==
old_tx['transaction_id'])
del transactions_out[idx]
except:
# old_tx not in the transactions_out list
pass
else:
# we'll use the other one
log.info((
"Setting tx '{}' to error due to another unconfirmed transaction"
"with nonce ({}) already existing in the system"
).format(old_tx['hash'],
transaction['nonce']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(
transaction['to_address'])
addresses_to_check.add(
transaction['from_address'])
# this case is actually pretty weird, so emptying the
# transactions_out so we restart the queue check
# completely
transactions_out = []
continue
else:
# well this is awkward! may as well let things go on in this case because
# it means a transaction in the nonce sequence is missing
pass
elif transaction['status'] == 'queued':
# then this and all the following transactions are now invalid
previous_error = True
log.info(
"Setting tx '{}' to error due to the nonce ({}) not matching the network ({})"
.format(transaction['hash'], transaction['nonce'],
nonce))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
else:
# this is a really weird state
# it's not clear what should be done here
log.error(
"Found unconfirmed transaction with out of order nonce for address: {}"
.format(ethereum_address))
return
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
cost = value + (gas * gas_price)
# check if the current balance is high enough to send to the network
if balance >= cost:
# check if gas price is high enough that it makes sense to send the transaction
safe_gas_price = parse_int(
await self.redis.get('gas_station_safelow_gas_price'))
if safe_gas_price and safe_gas_price > gas_price:
log.debug(
"Not queuing tx '{}' as current gas price would not support it"
.format(transaction['hash']))
# retry this address in a minute
manager_dispatcher.process_transaction_queue(
ethereum_address).delay(60)
# abort the rest of the processing after sending PNs for any "new" transactions
while transaction:
if transaction['status'] == 'new':
await self.update_transaction(
transaction['transaction_id'], 'queued')
transaction = transactions_out.pop(
) if transactions_out else None
break
# if so, send the transaction
# create the transaction
data = data_decoder(
transaction['data']) if transaction['data'] else b''
tx = create_transaction(nonce=transaction['nonce'],
value=value,
gasprice=gas_price,
startgas=gas,
to=transaction['to_address'],
data=data,
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
# make sure the signature was valid
if data_encoder(tx.sender) != ethereum_address:
# signature is invalid for the user
log.error(
"ERROR signature invalid for sender of tx: {}".
format(transaction['hash']))
log.error("queue: {}, db: {}, tx: {}".format(
ethereum_address, transaction['from_address'],
data_encoder(tx.sender)))
previous_error = True
addresses_to_check.add(transaction['to_address'])
await self.update_transaction(
transaction['transaction_id'], 'error')
continue
# send the transaction
try:
tx_encoded = encode_transaction(tx)
await self.eth.eth_sendRawTransaction(tx_encoded)
await self.update_transaction(
transaction['transaction_id'], 'unconfirmed')
except JsonRPCError as e:
# if something goes wrong with sending the transaction
# simply abort for now.
# TODO: depending on error, just break and queue to retry later
log.error(
"ERROR sending queued transaction: {}".format(
e.format()))
if e.message and (e.message.startswith(
"Transaction nonce is too low"
) or e.message.startswith(
"Transaction with the same hash was already imported"
)):
existing_tx = await self.eth.eth_getTransactionByHash(
transaction['hash'])
if existing_tx:
if existing_tx['blockNumber']:
await self.update_transaction(
transaction['transaction_id'],
'confirmed')
else:
await self.update_transaction(
transaction['transaction_id'],
'unconfirmed')
continue
previous_error = True
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# adjust the balance values for checking the other transactions
balance -= cost
if nonce == transaction['nonce']:
nonce += 1
continue
else:
# make sure the pending_balance would support this transaction
# otherwise there's no way this transaction will be able to
# be send, so trigger a failure on all the remaining transactions
async with self.db:
transactions_in = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE to_address = $1 "
"AND ("
"(status = 'new' OR status = 'queued' OR status = 'unconfirmed') "
"OR (status = 'confirmed' AND blocknumber > $2))",
ethereum_address, last_blocknumber or 0)
# TODO: test if loops in the queue chain are problematic
pending_received = sum(
(parse_int(p['value']) or 0) for p in transactions_in)
if balance + pending_received < cost:
previous_error = True
log.info(
"Setting tx '{}' to error due to insufficient pending balance"
.format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
else:
if any(t['blocknumber'] is not None
and t['blocknumber'] > last_blocknumber
for t in transactions_in):
addresses_to_check.add(ethereum_address)
# there's no reason to continue on here since all the
# following transaction in the queue cannot be processed
# until this one is
# but we still need to send PNs for any "new" transactions
while transaction:
if transaction['status'] == 'new':
await self.update_transaction(
transaction['transaction_id'], 'queued')
transaction = transactions_out.pop(
) if transactions_out else None
break
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
manager_dispatcher.process_transaction_queue(address)
if transactions_out:
manager_dispatcher.process_transaction_queue(ethereum_address)
async def send_transaction(self, *, tx, signature=None):
try:
tx = decode_transaction(tx)
except:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Invalid Transaction'
})
if is_transaction_signed(tx):
tx_sig = data_encoder(signature_from_transaction(tx))
if signature:
if tx_sig != signature:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: Signature in payload and signature of transaction do not match'
})
else:
signature = tx_sig
else:
if signature is None:
raise JsonRPCInvalidParamsError(data={
'id': 'missing_signature',
'message': 'Missing Signature'
})
if not validate_signature(signature):
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: {}'.format('Invalid length' if len(
signature) != 132 else 'Invalid hex value')
})
try:
sig = data_decoder(signature)
except Exception:
log.exception(
"Unexpected error decoding valid signature: {}".format(
signature))
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature'
})
add_signature_to_transaction(tx, sig)
# validate network id, if it's not for "all networks"
if tx.network_id is not None and self.network_id != tx.network_id:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_network_id',
'message': 'Invalid Network ID'
})
from_address = data_encoder(tx.sender)
to_address = data_encoder(tx.to)
# prevent spamming of transactions with the same nonce from the same sender
async with RedisLock("{}:{}".format(from_address, tx.nonce),
raise_when_locked=partial(
JsonRPCInvalidParamsError,
data={
'id': 'invalid_nonce',
'message': 'Nonce already used'
}),
ex=5):
# check for transaction overwriting
async with self.db:
existing = await self.db.fetchrow(
"SELECT * FROM transactions WHERE "
"from_address = $1 AND nonce = $2 AND "
"(status != 'error' or status is NULL)", from_address,
tx.nonce)
# disallow transaction overwriting when the gas is lower or the transaction is confirmed
if existing and (parse_int(existing['gas_price']) >= tx.gasprice
or existing['status'] == 'confirmed'):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_nonce',
'message': 'Nonce already used'
})
# make sure the account has enough funds for the transaction
network_balance, balance, _, _ = await self.get_balances(
from_address)
if existing:
balance += parse_int(existing['value']) + parse_int(
existing['gas']) * parse_int(existing['gas_price'])
if balance < (tx.value + (tx.startgas * tx.gasprice)):
raise JsonRPCInsufficientFundsError(
data={
'id': 'insufficient_funds',
'message': 'Insufficient Funds'
})
# validate the nonce (only necessary if tx doesn't already exist)
if not existing:
c_nonce = await self.get_transaction_count(from_address)
if tx.nonce < c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too low'
})
if tx.nonce > c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too high'
})
if tx.intrinsic_gas_used > tx.startgas:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_transaction',
'message':
'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'
.format(tx.intrinsic_gas_used, tx.startgas)
})
# now this tx fits enough of the criteria to allow it
# onto the transaction queue
tx_hash = calculate_transaction_hash(tx)
if existing:
log.info(
"Setting tx '{}' to error due to forced overwrite".format(
existing['hash']))
manager_dispatcher.update_transaction(
existing['transaction_id'], 'error')
data = data_encoder(tx.data)
if data and \
((data.startswith("0xa9059cbb") and len(data) == 138) or \
(data.startswith("0x23b872dd") and len(data) == 202)):
# check if the token is a known erc20 token
async with self.db:
erc20_token = await self.db.fetchrow(
"SELECT * FROM tokens WHERE contract_address = $1",
to_address)
else:
erc20_token = False
# add tx to database
async with self.db:
db_tx = await self.db.fetchrow(
"INSERT INTO transactions "
"(hash, from_address, to_address, nonce, "
"value, gas, gas_price, "
"data, v, r, s, "
"sender_toshi_id) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) "
"RETURNING transaction_id", tx_hash, from_address,
to_address, tx.nonce, hex(tx.value), hex(tx.startgas),
hex(tx.gasprice), data_encoder(tx.data), hex(tx.v),
hex(tx.r), hex(tx.s), self.user_toshi_id)
if erc20_token:
token_value = int(data[-64:], 16)
if data.startswith("0x23b872dd"):
erc20_from_address = "0x" + data[34:74]
erc20_to_address = "0x" + data[98:138]
else:
erc20_from_address = from_address
erc20_to_address = "0x" + data[34:74]
await self.db.execute(
"INSERT INTO token_transactions "
"(transaction_id, transaction_log_index, contract_address, from_address, to_address, value) "
"VALUES ($1, $2, $3, $4, $5, $6)",
db_tx['transaction_id'], 0,
erc20_token['contract_address'], erc20_from_address,
erc20_to_address, hex(token_value))
await self.db.commit()
# trigger processing the transaction queue
manager_dispatcher.process_transaction_queue(from_address)
# analytics
# use notification registrations to try find toshi ids for users
if self.user_toshi_id:
sender_toshi_id = self.user_toshi_id
else:
async with self.db:
sender_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", from_address)
async with self.db:
receiver_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", to_address)
self.track(sender_toshi_id, "Sent transaction")
# it doesn't make sense to add user agent here as we
# don't know the receiver's user agent
self.track(receiver_toshi_id,
"Received transaction",
add_user_agent=False)
return tx_hash
async def test_always_get_unconfirmed_push_notification(self, *, ethminer, push_client):
"""Tests that when tx's are send through our systems we always get
an unconfirmed push notification"""
# register for GCM PNs
body = {
"registration_id": TEST_GCM_ID,
"address": TEST_WALLET_ADDRESS
}
resp = await self.fetch_signed("/gcm/register", signing_key=TEST_ID_KEY, method="POST", body=body)
self.assertResponseCodeEqual(resp, 204, resp.body)
async with self.pool.acquire() as con:
rows = await con.fetch("SELECT * FROM notification_registrations WHERE toshi_id = $1", TEST_ID_ADDRESS)
self.assertIsNotNone(rows)
self.assertEqual(len(rows), 1)
# run this a bunch of times to see if
# we can expose any race conditions
for iteration in range(4):
if iteration > 2:
ethminer.pause()
value = 2821181018869341261
resp = await self.fetch("/tx/skel", method="POST", body={
"from": FAUCET_ADDRESS,
"to": TEST_WALLET_ADDRESS,
"value": value
})
self.assertResponseCodeEqual(resp, 200, resp.body)
body = json_decode(resp.body)
tx = sign_transaction(body['tx'], FAUCET_PRIVATE_KEY)
resp = await self.fetch("/tx", method="POST", body={
"tx": tx
})
self.assertResponseCodeEqual(resp, 200, resp.body)
tx_hash = json_decode(resp.body)['tx_hash']
tx = decode_transaction(tx)
self.assertEqual(tx_hash, data_encoder(tx.hash))
if iteration > 2:
await asyncio.sleep(5)
ethminer.start()
async with self.pool.acquire() as con:
rows = await con.fetch("SELECT * FROM transactions WHERE nonce = $1", tx.nonce)
self.assertEqual(len(rows), 1)
unconfirmed_count = 0
while True:
token, payload = await push_client.get()
self.assertEqual(token, TEST_GCM_ID)
message = parse_sofa_message(payload['message'])
self.assertIsInstance(message, SofaPayment)
self.assertEqual(message['value'], hex(value))
self.assertEqual(message['txHash'], tx_hash)
if message['status'] == "confirmed":
break
self.assertEqual(message['status'], "unconfirmed")
unconfirmed_count += 1
# when the tx is sent through our systems we should
# always get one unconfirmed notification, and we
# should never get more than one
self.assertEqual(unconfirmed_count, 1)
async def __call__(self,
*args,
startgas=None,
gasprice=20000000000,
value=0,
wait_for_confirmation=True):
# TODO: figure out if we can validate args
validated_args = []
for (type, name), arg in zip(
self.contract.translator.function_data[self.name]['signature'],
args):
if type == 'address' and isinstance(arg, str):
validated_args.append(data_decoder(arg))
elif (type.startswith("uint")
or type.startswith("int")) and isinstance(arg, str):
validated_args.append(int(arg, 16))
else:
validated_args.append(arg)
ethurl = get_url()
ethclient = JsonRPCClient(ethurl)
data = self.contract.translator.encode_function_call(
self.name, validated_args)
# TODO: figure out if there's a better way to tell if the function needs to be called via sendTransaction
if self.is_constant:
result = await ethclient.eth_call(from_address=self.from_address
or '',
to_address=self.contract.address,
data=data)
result = data_decoder(result)
if result:
decoded = self.contract.translator.decode_function_result(
self.name, result)
# decode string results
decoded = [
val.decode('utf-8')
if isinstance(val, bytes) and type == 'string' else val
for val, type in zip(
decoded, self.contract.translator.function_data[
self.name]['decode_types'])
]
# return the single value if there is only a single return value
if len(decoded) == 1:
return decoded[0]
return decoded
return None
else:
if self.from_address is None:
raise Exception(
"Cannot call non-constant function without a sender")
nonce = await ethclient.eth_getTransactionCount(self.from_address)
balance = await ethclient.eth_getBalance(self.from_address)
if startgas is None:
startgas = await ethclient.eth_estimateGas(
self.from_address,
self.contract.address,
data=data,
nonce=nonce,
value=value,
gasprice=gasprice)
if startgas == 50000000 or startgas is None:
raise Exception(
"Unable to estimate gas cost, possibly something wrong with the transaction arguments"
)
if balance < (startgas * gasprice):
raise Exception("Given account doesn't have enough funds")
tx = Transaction(nonce, gasprice, startgas, self.contract.address,
value, data, 0, 0, 0)
tx.sign(self.from_key)
tx_encoded = data_encoder(rlp.encode(tx, Transaction))
if self.return_raw_tx:
return tx_encoded
try:
tx_hash = await ethclient.eth_sendRawTransaction(tx_encoded)
except:
print(balance, startgas * gasprice, startgas)
raise
# wait for the contract to be deployed
if wait_for_confirmation:
print("waiting on transaction: {}".format(tx_hash))
starttime = time.time()
warnlevel = 0
while wait_for_confirmation:
resp = await ethclient.eth_getTransactionByHash(tx_hash)
if resp is None or resp['blockNumber'] is None:
await asyncio.sleep(0.1)
if resp is None and warnlevel == 0 and time.time(
) - starttime < 10:
print(
"WARNING: 10 seconds have passed and transaction is not showing as a pending transaction"
)
warnlevel = 1
elif resp is None and warnlevel == 1 and time.time(
) - starttime < 60:
print(
"WARNING: 60 seconds have passed and transaction is not showing as a pending transaction"
)
raise Exception(
"Unexpected error waiting for transaction to complete"
)
else:
receipt = await ethclient.eth_getTransactionReceipt(tx_hash
)
if 'status' in receipt and receipt['status'] != "0x1":
raise Exception(
"Transaction status returned {}".format(
receipt['status']))
break
# TODO: is it possible for non-const functions to have return types?
return tx_hash
async def from_source_code(cls,
sourcecode,
contract_name,
constructor_data=None,
*,
address=None,
deployer_private_key=None,
import_mappings=None,
libraries=None,
optimize=False,
deploy=True,
cwd=None,
wait_for_confirmation=True):
if deploy:
ethurl = get_url()
if address is None and deployer_private_key is None:
raise TypeError(
"requires either address or deployer_private_key")
if address is None and not isinstance(constructor_data,
(list, type(None))):
raise TypeError(
"must supply constructor_data as a list (hint: use [] if args should be empty)"
)
args = ['solc', '--combined-json', 'bin,abi']
if libraries:
args.extend([
'--libraries',
','.join(['{}:{}'.format(*library) for library in libraries])
])
if optimize:
args.append('--optimize')
if import_mappings:
args.extend([
"{}={}".format(path, mapping)
for path, mapping in import_mappings
])
# check if sourcecode is actually a filename
if cwd:
filename = os.path.join(cwd, sourcecode)
else:
filename = sourcecode
if os.path.exists(filename):
args.append(filename)
sourcecode = None
else:
filename = '<stdin>'
process = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
output, stderrdata = process.communicate(input=sourcecode)
try:
output = json_decode(output)
except json.JSONDecodeError:
if output and stderrdata:
output += b'\n' + stderrdata
elif stderrdata:
output = stderrdata
raise Exception("Failed to compile source: {}\n{}\n{}".format(
filename, ' '.join(args), output.decode('utf-8')))
try:
contract = output['contracts']['{}:{}'.format(
filename, contract_name)]
except KeyError:
print(output)
raise
abi = json_decode(contract['abi'])
# deploy contract
translator = ContractTranslator(abi)
# fix things that don't have a constructor
if not deploy:
return Contract(abi=abi, address=address, translator=translator)
ethclient = JsonRPCClient(ethurl)
if address is not None:
# verify there is code at the given address
for i in range(10):
code = await ethclient.eth_getCode(address)
if code == "0x":
await asyncio.sleep(1)
continue
break
else:
raise Exception("No code found at given address")
return Contract(abi=abi, address=address, translator=translator)
try:
bytecode = data_decoder(contract['bin'])
except binascii.Error:
print(contract['bin'])
raise
if constructor_data is not None:
constructor_call = translator.encode_constructor_arguments(
constructor_data)
bytecode += constructor_call
if isinstance(deployer_private_key, str):
deployer_private_key = data_decoder(deployer_private_key)
deployer_address = private_key_to_address(deployer_private_key)
nonce = await ethclient.eth_getTransactionCount(deployer_address)
balance = await ethclient.eth_getBalance(deployer_address)
gasprice = 20000000000
value = 0
startgas = await ethclient.eth_estimateGas(deployer_address,
'',
data=bytecode,
nonce=nonce,
value=0,
gasprice=gasprice)
if balance < (startgas * gasprice):
raise Exception("Given account doesn't have enough funds")
tx = Transaction(nonce, gasprice, startgas, '', value, bytecode, 0, 0,
0)
tx.sign(deployer_private_key)
tx_encoded = data_encoder(rlp.encode(tx, Transaction))
contract_address = data_encoder(tx.creates)
tx_hash = await ethclient.eth_sendRawTransaction(tx_encoded)
# wait for the contract to be deployed
while wait_for_confirmation:
resp = await ethclient.eth_getTransactionByHash(tx_hash)
if resp is None or resp['blockNumber'] is None:
await asyncio.sleep(0.1)
else:
code = await ethclient.eth_getCode(contract_address)
if code == '0x':
raise Exception(
"Failed to deploy contract: resulting address '{}' has no code"
.format(contract_address))
break
return Contract(abi=abi,
address=contract_address,
translator=translator,
creation_tx_hash=tx_hash)
async def _process_transaction_queue(self, ethereum_address):
log.info("processing tx queue for {}".format(ethereum_address))
# check for un-scheduled transactions
async with self.db:
# get the last block number to use in ethereum calls
# to avoid race conditions in transactions being confirmed
# on the network before the block monitor sees and updates them in the database
last_blocknumber = (
await
self.db.fetchval("SELECT blocknumber FROM last_blocknumber"))
transactions_out = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE from_address = $1 "
"AND (status is NULL OR status = 'queued') "
"AND r IS NOT NULL "
# order by nonce reversed so that .pop() can
# be used in the loop below
"ORDER BY nonce DESC",
ethereum_address)
# any time the state of a transaction is changed we need to make
# sure those changes cascade down to the receiving address as well
# this keeps a list of all the receiving addresses that need to be
# checked after the current address's queue has been processed
addresses_to_check = set()
if transactions_out:
# TODO: make sure the block number isn't too far apart from the current
# if this is the case then we should just come back later!
# get the current network balance for this address
balance = await self.eth.eth_getBalance(ethereum_address,
block=last_blocknumber
or "latest")
# get the unconfirmed_txs
async with self.db:
unconfirmed_txs = await self.db.fetch(
"SELECT nonce, value, gas, gas_price FROM transactions "
"WHERE from_address = $1 "
"AND (status = 'unconfirmed' "
"OR (status = 'confirmed' AND blocknumber > $2)) "
"ORDER BY nonce", ethereum_address, last_blocknumber or 0)
if unconfirmed_txs:
nonce = unconfirmed_txs[-1]['nonce'] + 1
balance -= sum(
parse_int(tx['value']) +
(parse_int(tx['gas']) * parse_int(tx['gas_price']))
for tx in unconfirmed_txs)
else:
# use the nonce from the network
nonce = await self.eth.eth_getTransactionCount(
ethereum_address, block=last_blocknumber or "latest")
# marker for whether a previous transaction had an error (signaling
# that all the following should also be an error
previous_error = False
# for each one, check if we can schedule them yet
while transactions_out:
transaction = transactions_out.pop()
# if there was a previous error in the queue, abort!
if previous_error:
log.info("Setting tx '{}' to error due to previous error".
format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# make sure the nonce is still valid
if nonce != transaction['nonce']:
# then this and all the following transactions are now invalid
previous_error = True
log.info(
"Setting tx '{}' to error due to the nonce ({}) not matching the network ({})"
.format(transaction['hash'], transaction['nonce'],
nonce))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
cost = value + (gas * gas_price)
# check if the current balance is high enough to send to the network
if balance >= cost:
# if so, send the transaction
# create the transaction
data = data_decoder(
transaction['data']) if transaction['data'] else b''
tx = create_transaction(nonce=nonce,
value=value,
gasprice=gas_price,
startgas=gas,
to=transaction['to_address'],
data=data,
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
# make sure the signature was valid
if data_encoder(tx.sender) != ethereum_address:
# signature is invalid for the user
log.error(
"ERROR signature invalid for sender of tx: {}".
format(transaction['hash']))
log.error("queue: {}, db: {}, tx: {}".format(
ethereum_address, transaction['from_address'],
data_encoder(tx.sender)))
previous_error = True
addresses_to_check.add(transaction['to_address'])
await self.update_transaction(
transaction['transaction_id'], 'error')
continue
# send the transaction
try:
tx_encoded = encode_transaction(tx)
await self.eth.eth_sendRawTransaction(tx_encoded)
await self.update_transaction(
transaction['transaction_id'], 'unconfirmed')
except JsonRPCError as e:
# if something goes wrong with sending the transaction
# simply abort for now.
# TODO: depending on error, just break and queue to retry later
log.error(
"ERROR sending queued transaction: {}".format(
e.format()))
previous_error = True
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
# adjust the balance values for checking the other transactions
balance -= cost
nonce += 1
continue
else:
# make sure the pending_balance would support this transaction
# otherwise there's no way this transaction will be able to
# be send, so trigger a failure on all the remaining transactions
async with self.db:
transactions_in = await self.db.fetch(
"SELECT * FROM transactions "
"WHERE to_address = $1 "
"AND ("
"(status is NULL OR status = 'queued' OR status = 'unconfirmed') "
"OR (status = 'confirmed' AND blocknumber > $2))",
ethereum_address, last_blocknumber or 0)
# TODO: test if loops in the queue chain are problematic
pending_received = sum(
(parse_int(p['value']) or 0) for p in transactions_in)
if balance + pending_received < cost:
previous_error = True
log.info(
"Setting tx '{}' to error due to insufficient pending balance"
.format(transaction['hash']))
await self.update_transaction(
transaction['transaction_id'], 'error')
addresses_to_check.add(transaction['to_address'])
continue
else:
if any(t['blocknumber'] is not None
and t['blocknumber'] > last_blocknumber
for t in transactions_in):
addresses_to_check.add(ethereum_address)
# there's no reason to continue on here since all the
# following transaction in the queue cannot be processed
# until this one is
# but we still need to send PNs for any "new" transactions
while transaction:
if transaction['status'] is None:
await self.update_transaction(
transaction['transaction_id'], 'queued')
transaction = transactions_out.pop(
) if transactions_out else None
break
for address in addresses_to_check:
# make sure we don't try process any contract deployments
if address != "0x":
self.tasks.process_transaction_queue(address)
if transactions_out:
self.tasks.process_transaction_queue(ethereum_address)
async def send_transaction(self, *, tx, signature=None):
try:
tx = decode_transaction(tx)
except:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Invalid Transaction'
})
if is_transaction_signed(tx):
tx_sig = data_encoder(signature_from_transaction(tx))
if signature:
if tx_sig != signature:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: Signature in payload and signature of transaction do not match'
})
else:
signature = tx_sig
else:
if signature is None:
raise JsonRPCInvalidParamsError(data={
'id': 'missing_signature',
'message': 'Missing Signature'
})
if not validate_signature(signature):
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_signature',
'message':
'Invalid Signature: {}'.format('Invalid length' if len(
signature) != 132 else 'Invalid hex value')
})
try:
sig = data_decoder(signature)
except Exception:
log.exception(
"Unexpected error decoding valid signature: {}".format(
signature))
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature'
})
add_signature_to_transaction(tx, sig)
# validate network id, if it's not for "all networks"
if tx.network_id is not None and self.network_id != tx.network_id:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_network_id',
'message': 'Invalid Network ID'
})
from_address = data_encoder(tx.sender)
to_address = data_encoder(tx.to)
# prevent spamming of transactions with the same nonce from the same sender
with RedisLock(self.redis,
"{}:{}".format(from_address, tx.nonce),
raise_when_locked=partial(JsonRPCInvalidParamsError,
data={
'id':
'invalid_nonce',
'message':
'Nonce already used'
}),
ex=5):
# disallow transaction overwriting for known transactions
async with self.db:
existing = await self.db.fetchrow(
"SELECT * FROM transactions WHERE "
"from_address = $1 AND nonce = $2 AND status != $3",
from_address, tx.nonce, 'error')
if existing:
# debugging checks
existing_tx = await self.eth.eth_getTransactionByHash(
existing['hash'])
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_nonce',
'message': 'Nonce already used'
})
# make sure the account has enough funds for the transaction
network_balance, balance, _, _ = await self.get_balances(
from_address)
#log.info("Attempting to send transaction\nHash: {}\n{} -> {}\nValue: {} + {} (gas) * {} (startgas) = {}\nSender's Balance {} ({} unconfirmed)".format(
# calculate_transaction_hash(tx), from_address, to_address, tx.value, tx.startgas, tx.gasprice, tx.value + (tx.startgas * tx.gasprice), network_balance, balance))
if balance < (tx.value + (tx.startgas * tx.gasprice)):
raise JsonRPCInsufficientFundsError(
data={
'id': 'insufficient_funds',
'message': 'Insufficient Funds'
})
# validate the nonce
c_nonce = await self.get_transaction_count(from_address)
if tx.nonce < c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too low'
})
if tx.nonce > c_nonce:
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_nonce',
'message': 'Provided nonce is too high'
})
if tx.intrinsic_gas_used > tx.startgas:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_transaction',
'message':
'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'
.format(tx.intrinsic_gas_used, tx.startgas)
})
# now this tx fits enough of the criteria to allow it
# onto the transaction queue
tx_hash = calculate_transaction_hash(tx)
# add tx to database
async with self.db:
await self.db.execute(
"INSERT INTO transactions "
"(hash, from_address, to_address, nonce, "
"value, gas, gas_price, "
"data, v, r, s, "
"sender_toshi_id) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
tx_hash, from_address, to_address, tx.nonce, hex(tx.value),
hex(tx.startgas), hex(tx.gasprice), data_encoder(tx.data),
hex(tx.v), hex(tx.r), hex(tx.s), self.user_toshi_id)
await self.db.commit()
# trigger processing the transaction queue
self.tasks.process_transaction_queue(from_address)
# analytics
# use notification registrations to try find toshi ids for users
if self.user_toshi_id:
sender_toshi_id = self.user_toshi_id
else:
async with self.db:
sender_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", from_address)
async with self.db:
receiver_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1", to_address)
self.track(sender_toshi_id, "Sent transaction")
# it doesn't make sense to add user agent here as we
# don't know the receiver's user agent
self.track(receiver_toshi_id,
"Received transaction",
add_user_agent=False)
return tx_hash
def do_POST(self):
# TODO: figure out why read is blocking here
data = self.rfile.read(len(self.rfile.peek()))
data = data.decode('utf-8')
data = json.loads(data)
if self.path == "/v1/tx/skel":
gas_price = parse_int(
data['gas_price']) if 'gas_price' in data else DEFAULT_GASPRICE
gas = parse_int(data['gas']) if 'gas' in data else DEFAULT_STARTGAS
nonce = parse_int(data['nonce']) if 'nonce' in data else 0
if 'value' not in data or 'from' not in data or 'to' not in data:
self.write_data(
400, {
'errors': [{
'id': 'bad_arguments',
'message': 'Bad Arguments'
}]
})
return
value = parse_int(data['value'])
to_address = data['to']
from_address = data['from']
if not validate_address(to_address):
self.write_data(
400, {
'errors': [{
'id': 'invalid_to_address',
'message': 'Invalid To Address'
}]
})
return
if not validate_address(from_address):
self.write_data(
400, {
'errors': [{
'id': 'invalid_from_address',
'message': 'Invalid From Address'
}]
})
return
tx = create_transaction(nonce=nonce,
gasprice=gas_price,
startgas=gas,
to=to_address,
value=value)
transaction = encode_transaction(tx)
self.write_data(
200, {
"tx_data": {
"nonce": hex(nonce),
"from": from_address,
"to": to_address,
"value": hex(value),
"startGas": hex(gas),
"gasPrice": hex(gas_price)
},
"tx": transaction
})
elif self.path == "/v1/tx":
tx = decode_transaction(data['tx'])
if 'signature' in data:
sig = data_decoder(data['signature'])
add_signature_to_transaction(tx, sig)
self.write_data(200, {"tx_hash": data_encoder(tx.hash)})
else:
self.write_data(404)
