async def test_get_balance_with_error_txs(self):
tx1_hash = '0x2f321aa116146a9bc62b61c76508295f708f42d56340c9e613ebfc27e33f240c'
tx2_hash = '0x2f321aa116146a9bc62b61c76508295f708f42d56340c9e613ebfc27e33f240d'
tx3_hash = '0x2f321aa116146a9bc62b61c76508295f708f42d56340c9e613ebfc27e33f240e'
addr = '0x39bf9e501e61440b4b268d7b2e9aa2458dd201bb'
val = 761751855997712
await self.faucet(addr, val)
async with self.pool.acquire() as con:
await con.execute(
"INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price) "
"VALUES ($1, $2, $3, $4, $5, $6, $7)", tx1_hash,
FAUCET_ADDRESS, addr, 0, val, DEFAULT_STARTGAS,
DEFAULT_GASPRICE)
await con.execute(
"INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, status) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", tx2_hash,
FAUCET_ADDRESS, addr, 1, val, DEFAULT_STARTGAS,
DEFAULT_GASPRICE, 'unconfirmed')
await con.execute(
"INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, status) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", tx3_hash,
FAUCET_ADDRESS, addr, 2, val, DEFAULT_STARTGAS,
DEFAULT_GASPRICE, 'error')
resp = await self.fetch('/balance/{}'.format(addr))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), val)
self.assertEqual(parse_int(data['unconfirmed_balance']), val * 3)
async def test_create_and_send_transaction_with_max_value_with_pending_balance(
self, *, monitor):
await self.faucet(TEST_ADDRESS, 10 * 10**18)
tx_hash_1 = await self.send_tx(TEST_PRIVATE_KEY, TEST_ADDRESS_2,
5 * 10**18)
resp = await self.fetch("/tx/skel",
method="POST",
body={
"from": TEST_ADDRESS,
"to": TEST_ADDRESS_2,
"value": "max"
})
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx_hash_2 = await self.sign_and_send_tx(TEST_PRIVATE_KEY, body['tx'])
await self.wait_on_tx_confirmation(tx_hash_1)
await self.wait_on_tx_confirmation(tx_hash_2)
await monitor.block_check()
await asyncio.sleep(0.1)
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), 0)
self.assertEqual(parse_int(data['unconfirmed_balance']), 0)
async def test_send_transaction_with_unconfirmed_funds(self, *, ethminer):
"""Tests that someone can create a tx with unconfirmed funds"""
# make sure no blocks are mined
ethminer.pause()
# send funds to address1
val1 = 10 ** 18
resp = await self.fetch("/tx/skel", method="POST", body={
"from": FAUCET_ADDRESS,
"to": TEST_ADDRESS_1,
"value": val1 # 1 eth
})
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx = sign_transaction(body['tx'], FAUCET_PRIVATE_KEY)
resp = await self.fetch("/tx", method="POST", body={
"tx": tx
})
self.assertEqual(resp.code, 200, resp.body)
tx1_hash = json_decode(resp.body)['tx_hash']
# send transaction from address1
val2 = 2 * 10 ** 17 # 0.2 eth
resp = await self.fetch("/tx/skel", method="POST", body={
"from": TEST_ADDRESS_1,
"to": TEST_ADDRESS_2,
"value": val2
})
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx = sign_transaction(body['tx'], TEST_PRIVATE_KEY_1)
resp = await self.fetch("/tx", method="POST", body={
"tx": tx
})
self.assertEqual(resp.code, 200, resp.body)
tx2_hash = json_decode(resp.body)['tx_hash']
await asyncio.sleep(1)
# make sure 2nd transaction is 'queued'
async with self.pool.acquire() as con:
row = await con.fetchrow("SELECT * FROM transactions WHERE from_address = $1", TEST_ADDRESS_1)
self.assertIsNotNone(row)
self.assertEqual(row['status'], 'queued')
# make sure balance adjusts for queued items
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS_1))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), 0)
self.assertEqual(parse_int(data['unconfirmed_balance']), val1 - (val2 + (DEFAULT_STARTGAS * DEFAULT_GASPRICE)))
ethminer.start()
await self.wait_on_tx_confirmation(tx1_hash)
await self.wait_on_tx_confirmation(tx2_hash)
await asyncio.sleep(1)
async def test_create_and_send_transaction(self):
val = 10 ** 10
body = {
"from": FAUCET_ADDRESS,
"to": TEST_ADDRESS,
"value": val
}
resp = await self.fetch("/tx/skel", method="POST", body=body)
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx = sign_transaction(body['tx'], FAUCET_PRIVATE_KEY)
body = {
"tx": tx
}
resp = await self.fetch("/tx", method="POST", body=body)
self.assertEqual(resp.code, 200, resp.body)
# ensure we get a tracking events
self.assertEqual((await self.next_tracking_event())[0], None)
body = json_decode(resp.body)
tx_hash = body['tx_hash']
tx = decode_transaction(tx)
self.assertEqual(tx_hash, data_encoder(tx.hash))
async with self.pool.acquire() as con:
rows = await con.fetch("SELECT * FROM transactions WHERE nonce = $1", tx.nonce)
self.assertEqual(len(rows), 1)
# wait for a push notification
await self.wait_on_tx_confirmation(tx_hash)
while True:
async with self.pool.acquire() as con:
row = await con.fetchrow("SELECT * FROM transactions WHERE nonce = $1", tx.nonce)
if row['status'] == 'confirmed':
break
# make sure updated field is updated
self.assertGreater(row['updated'], row['created'])
# make sure balance is returned correctly
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), val)
self.assertEqual(parse_int(data['unconfirmed_balance']), val)
async def test_get_balance_without_queued_items(self):
tx0_hash = '0x2f321aa116146a9bc62b61c76508295f708f42d56340c9e613ebfc27e33f240b'
tx1_hash = '0x2f321aa116146a9bc62b61c76508295f708f42d56340c9e613ebfc27e33f240c'
tx2_hash = '0x2f321aa116146a9bc62b61c76508295f708f42d56340c9e613ebfc27e33f240d'
tx3_hash = '0x2f321aa116146a9bc62b61c76508295f708f42d56340c9e613ebfc27e33f240e'
addr = '0x39bf9e501e61440b4b268d7b2e9aa2458dd201bb'
addr2 = '0x66c3dcc38542467eb6ddeef194add1d9eaaf05e0'
val = 76175185599771243
sent_val = 2 * 10 ** 10
await self.faucet(addr, val)
async with self.pool.acquire() as con:
await con.execute(
"INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, status) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
tx0_hash, addr, addr2, 0, hex(sent_val),
hex(DEFAULT_STARTGAS), hex(DEFAULT_GASPRICE), 'confirmed')
await con.execute(
"INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price) "
"VALUES ($1, $2, $3, $4, $5, $6, $7)",
tx1_hash, addr, addr2, 0, hex(sent_val),
hex(DEFAULT_STARTGAS), hex(DEFAULT_GASPRICE))
await con.execute(
"INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, status) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
tx2_hash, addr, addr2, 1, hex(sent_val),
hex(DEFAULT_STARTGAS), hex(DEFAULT_GASPRICE),
'queued')
await con.execute(
"INSERT INTO transactions (hash, from_address, to_address, nonce, value, gas, gas_price, status) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
tx3_hash, addr, addr2, 2, hex(sent_val),
hex(DEFAULT_STARTGAS), hex(DEFAULT_GASPRICE),
'unconfirmed')
resp = await self.fetch('/{}'.format(addr))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), val)
self.assertEqual(parse_int(data['unconfirmed_balance']),
val - (sent_val + (DEFAULT_STARTGAS * DEFAULT_GASPRICE)))
resp = await self.fetch('/{}'.format(addr2))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), 0)
self.assertEqual(parse_int(data['unconfirmed_balance']), sent_val)
async def get_balances(self, eth_address, include_queued=True):
"""Gets the confirmed balance of the eth address from the ethereum network
and adjusts the value based off any pending transactions.
Returns 4 values as a tuple:
- the confirmed (network) balance
- the balance adjusted for any pending transactions
- the total value of pending transactions sent from the given address
- the total value of pending transactions sent to the given address
"""
async with self.db:
# get the last block number to use in ethereum calls
# to avoid race conditions in transactions being confirmed
# on the network before the block monitor sees and updates them in the database
block = (
await
self.db.fetchval("SELECT blocknumber FROM last_blocknumber"))
pending_sent = await self.db.fetch(
"SELECT hash, value, gas, gas_price, status FROM transactions "
"WHERE from_address = $1 "
"AND ("
"((status != 'error' AND status != 'confirmed') OR status = 'new') "
"OR (status = 'confirmed' AND blocknumber > $2))", eth_address,
block or 0)
pending_received = await self.db.fetch(
"SELECT hash, value, status FROM transactions "
"WHERE to_address = $1 "
"AND ("
"((status != 'error' AND status != 'confirmed') OR status = 'new') "
"OR (status = 'confirmed' AND blocknumber > $2))", eth_address,
block or 0)
pending_sent = sum(
parse_int(p['value']) +
(parse_int(p['gas']) * parse_int(p['gas_price']))
for p in pending_sent
if include_queued or p['status'] == 'unconfirmed')
pending_received = sum(
parse_int(p['value']) for p in pending_received
if include_queued or p['status'] == 'unconfirmed')
confirmed_balance = await self.eth.eth_getBalance(eth_address,
block=block
or "latest")
balance = (confirmed_balance + pending_received) - pending_sent
return confirmed_balance, balance, pending_sent, pending_received
async def _run_erc20_health_check(self):
log.info("running erc20 health check")
async with self.pool.acquire() as con:
token_balances = await con.fetch("SELECT * FROM token_balances")
bad = 0
requests = []
last_execute = 0
bulk = self.eth.bulk()
for token in token_balances:
contract_address = token['contract_address']
data = "0x70a08231000000000000000000000000" + token['eth_address'][
2:]
f = bulk.eth_call(to_address=contract_address, data=data)
requests.append(
(contract_address, token['eth_address'], f, token['value']))
if len(requests) >= last_execute + 500:
await bulk.execute()
bulk = self.eth.bulk()
last_execute = len(requests)
if len(requests) > last_execute:
await bulk.execute()
bad_data = {}
for contract_address, eth_address, f, db_value in requests:
if not f.done():
log.warning("future not done when checking erc20 cache")
continue
try:
value = f.result()
except:
log.exception("error getting erc20 value {}:{}".format(
contract_address, eth_address))
continue
if parse_int(value) != parse_int(db_value):
bad += 1
bad_data.setdefault(eth_address, set()).add(contract_address)
if bad > 0:
log.warning(
"Found {}/{} bad ERC20 caches over {} addresses".format(
bad, len(token_balances), len(bad_data)))
for eth_address in bad_data:
erc20_dispatcher.update_token_cache("*", eth_address)
await asyncio.sleep(15) # don't overload things
async def test_get_balance(self):
addr = '0x39bf9e501e61440b4b268d7b2e9aa2458dd201bb'
val = 761751855997712
await self.faucet(addr, val)
resp = await self.fetch('/balance/{}'.format(addr))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), val)
self.assertEqual(parse_int(data['unconfirmed_balance']), val)
async def _eth_getLogs_with_block_number_validation(self, kwargs):
req_start = time.time()
from_block = parse_int(kwargs.get('fromBlock', None))
to_block = parse_int(kwargs.get('toBlock', None))
while True:
bulk = self.bulk()
bn_future = bulk.eth_blockNumber()
lg_future = bulk._fetch("eth_getLogs", [kwargs])
await bulk.execute()
bn = bn_future.result()
if (from_block and bn < from_block) or (to_block
and bn < to_block):
if self.should_retry and time.time(
) - req_start < self._request_timeout:
await asyncio.sleep(random.random())
continue
raise JsonRPCError(None, -32000, "Unknown block number", None)
return lg_future.result()
async def get_reports(request, conf, current_user):
page = parse_int(request.args.get('page', None)) or 1
if page < 1:
page = 1
limit = 10
offset = (page - 1) * limit
sql = ("SELECT * FROM reports "
"ORDER BY report_id DESC "
"OFFSET $1 LIMIT $2")
args = [offset, limit]
count_sql = ("SELECT COUNT(*) FROM reports")
count_args = []
async with conf.db.id.acquire() as con:
rows = await con.fetch(sql, *args)
count = await con.fetchrow(count_sql, *count_args)
reports = []
for row in rows:
async with conf.db.id.acquire() as con:
reporter = await con.fetchrow(
"SELECT * FROM users WHERE toshi_id = $1",
row['reporter_toshi_id'])
reportee = await con.fetchrow(
"SELECT * FROM users WHERE toshi_id = $1",
row['reportee_toshi_id'])
reporter = fix_avatar_for_user(conf.urls.id, dict(reporter))
reportee = fix_avatar_for_user(conf.urls.id, dict(reportee))
reports.append({
'reporter': reporter,
'reportee': reportee,
'details': row['details'],
'date': row['date']
})
total_pages = (count['count'] // limit) + (0 if count['count'] %
limit == 0 else 1)
def get_qargs(page=page, as_list=False, as_dict=False):
qargs = {'page': page}
if as_dict:
return qargs
if as_list:
return qargs.items()
return urlencode(qargs)
return html(await env.get_template("reports.html").render_async(
reports=reports,
current_user=current_user,
environment=conf.name,
page="reports",
total=count['count'],
total_pages=total_pages,
current_page=page,
get_qargs=get_qargs))
async def block_check(self):
while not self._shutdown:
block = await self.eth.eth_getBlockByNumber(self.last_block_number + 1)
if block:
if self._lastlog + 1800 < asyncio.get_event_loop().time():
self._lastlog = asyncio.get_event_loop().time()
log.info("Processing block {}".format(block['number']))
if block['logsBloom'] != "0x" + ("0" * 512):
logs_list = await self.eth.eth_getLogs(fromBlock=block['number'],
toBlock=block['number'])
logs = {}
for _log in logs_list:
if _log['transactionHash'] not in logs:
logs[_log['transactionHash']] = [_log]
else:
logs[_log['transactionHash']].append(_log)
else:
logs_list = []
logs = {}
for tx in block['transactions']:
# send notifications to sender and reciever
if tx['hash'] in logs:
tx['logs'] = logs[tx['hash']]
await self.process_transaction(tx)
if logs_list:
# send notifications for anyone registered
async with self.pool.acquire() as con:
for event in logs_list:
for topic in event['topics']:
filters = await con.fetch(
"SELECT * FROM filter_registrations WHERE contract_address = $1 AND topic_id = $2",
event['address'], topic)
for filter in filters:
eth_dispatcher.send_filter_notification(
filter['filter_id'], filter['topic'], event['data'])
# update the latest block number, only if it is larger than the
# current block number.
block_number = parse_int(block['number'])
if self.last_block_number < block_number:
self.last_block_number = block_number
async with self.pool.acquire() as con:
await con.execute("UPDATE last_blocknumber SET blocknumber = $1 "
"WHERE blocknumber < $1",
block_number)
collectibles_dispatcher.notify_new_block(block_number)
else:
break
self._block_checking_process = None
async def get_txs(request, conf, user):
page = parse_int(request.args.get('page', None)) or 1
if page < 1:
page = 1
limit = 10
offset = (page - 1) * limit
where_clause = ''
filters = [
f for f in request.args.getlist('filter', [])
if f in ['confirmed', 'unconfirmed', 'error']
]
if filters:
where_clause = "WHERE " + " OR ".join("status = '{}'".format(f)
for f in filters)
if 'unconfirmed' in filters:
where_clause += " OR status IS NULL"
async with conf.db.eth.acquire() as con:
rows = await con.fetch(
"SELECT * FROM transactions {} ORDER BY created DESC OFFSET $1 LIMIT $2"
.format(where_clause), offset, limit)
count = await con.fetchrow(
"SELECT COUNT(*) FROM transactions {}".format(where_clause))
txs = []
for row in rows:
tx = dict(row)
tx['from_user'] = await get_toshi_user_from_payment_address(
conf, tx['from_address'])
tx['to_user'] = await get_toshi_user_from_payment_address(
conf, tx['to_address'])
txs.append(tx)
total_pages = (count['count'] // limit) + (0 if count['count'] %
limit == 0 else 1)
def get_qargs(page=page, filters=filters, as_list=False, as_dict=False):
qargs = {'page': page}
if filters:
qargs['filter'] = filters
if as_dict:
return qargs
if as_list:
return qargs.items()
return urlencode(qargs, True)
return html(await env.get_template("txs.html").render_async(
txs=txs,
current_user=user,
environment=conf.name,
page="txs",
total=count['count'],
total_pages=total_pages,
current_page=page,
active_filters=filters,
get_qargs=get_qargs))
async def update_token_cache(self, contract_address, *eth_addresses):
if len(eth_addresses) == 0:
return
async with self.db:
if contract_address == "*":
tokens = await self.db.fetch(
"SELECT contract_address FROM tokens")
else:
tokens = [{'contract_address': contract_address}]
futures = []
for token in tokens:
for address in eth_addresses:
# data for `balanceOf(address)`
data = "0x70a08231000000000000000000000000" + address[2:]
f = asyncio.ensure_future(
self.eth.eth_call(to_address=token['contract_address'],
data=data))
futures.append((token['contract_address'], address, f))
# wait for all the jsonrpc calls to finish
await asyncio.gather(*[f[2] for f in futures], return_exceptions=True)
bulk_update = []
bulk_delete = []
for contract_address, eth_address, f in futures:
try:
value = f.result()
# value of "0x" means something failed with the contract call
if value == "0x0000000000000000000000000000000000000000000000000000000000000000" or value == "0x":
if value == "0x":
log.warning(
"calling balanceOf for contract {} failed".format(
contract_address))
bulk_delete.append((contract_address, eth_address))
else:
value = hex(
parse_int(value)) # remove hex padding of value
bulk_update.append((contract_address, eth_address, value))
except:
log.exception(
"WARNING: failed to update token cache of '{}' for address: {}"
.format(contract_address, eth_address))
continue
async with self.db:
await self.db.executemany(
"INSERT INTO token_balances (contract_address, eth_address, value) VALUES ($1, $2, $3) "
"ON CONFLICT (contract_address, eth_address) DO UPDATE set value = EXCLUDED.value",
bulk_update)
await self.db.executemany(
"DELETE FROM token_balances WHERE contract_address = $1 AND eth_address = $2",
bulk_delete)
await self.db.commit()
def to_eth(wei, points=18):
wei = str(parse_int(wei))
pad = 18 - len(wei)
if pad < 0:
eth = wei[:abs(pad)] + "." + wei[abs(pad):abs(pad) + points]
else:
eth = "0." + wei.zfill(18)[:points]
while eth.endswith("0"):
eth = eth[:-1]
if eth.endswith("."):
eth += "0"
return eth
async def test_empty_account(self):
"""Makes sure an account can be emptied completely"""
val = 10**16
default_fees = DEFAULT_STARTGAS * DEFAULT_GASPRICE
tx_hash = await self.send_tx(FAUCET_PRIVATE_KEY, TEST_ADDRESS, val)
tx = await self.wait_on_tx_confirmation(tx_hash)
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), val)
self.assertEqual(parse_int(data['unconfirmed_balance']), val)
resp = await self.fetch("/tx/skel",
method="POST",
body={
"from": TEST_ADDRESS,
"to": FAUCET_ADDRESS,
"value": val - default_fees
})
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx = sign_transaction(body['tx'], TEST_PRIVATE_KEY)
resp = await self.fetch("/tx", method="POST", body={"tx": tx})
self.assertEqual(resp.code, 200, resp.body)
body = json_decode(resp.body)
tx_hash = body['tx_hash']
# wait for a push notification
tx = await self.wait_on_tx_confirmation(tx_hash)
# make sure balance is returned correctly (and is 0)
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), 0)
self.assertEqual(parse_int(data['unconfirmed_balance']), 0)
async def test_create_and_send_transaction_with_max_value_to_contract(self, *, monitor, parity):
contract = await Contract.from_source_code(
SPLITTER_CONTRACT.encode('utf-8'), "Splitter",
constructor_data=[[TEST_ADDRESS, TEST_ADDRESS_2, TEST_ADDRESS_3, TEST_ADDRESS_4]],
deployer_private_key=FAUCET_PRIVATE_KEY)
await self.faucet(TEST_ADDRESS, 9 * 10 ** 18)
resp = await self.fetch("/tx/skel", method="POST", body={
"from": TEST_ADDRESS,
"to": contract.address,
"value": "max"
})
self.assertEqual(resp.code, 200)
body = json_decode(resp.body)
tx_hash = await self.sign_and_send_tx(TEST_PRIVATE_KEY, body['tx'])
await self.wait_on_tx_confirmation(tx_hash)
await monitor.block_check()
await asyncio.sleep(0.1)
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), 0)
self.assertEqual(parse_int(data['unconfirmed_balance']), 0)
tx_hash = await contract.withdraw.set_sender(FAUCET_PRIVATE_KEY)()
await monitor.block_check()
await asyncio.sleep(0.1)
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS_2))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
expected_balance = parse_int(data['confirmed_balance'])
print(expected_balance)
self.assertNotEqual(expected_balance, 0)
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS_2))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), expected_balance)
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS_3))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), expected_balance)
resp = await self.fetch('/balance/{}'.format(TEST_ADDRESS_4))
self.assertEqual(resp.code, 200)
data = json_decode(resp.body)
self.assertEqual(parse_int(data['confirmed_balance']), expected_balance)
async def _list_payment_updates(self, address, start_time, end_time=None):
if end_time is None:
end_time = datetime.utcnow()
elif not isinstance(end_time, datetime):
end_time = datetime.utcfromtimestamp(end_time)
if not isinstance(start_time, datetime):
start_time = datetime.utcfromtimestamp(start_time)
async with self.db:
txs = await self.db.fetch(
"SELECT * FROM transactions WHERE "
"(from_address = $1 OR to_address = $1) AND "
"updated > $2 AND updated < $3"
"ORDER BY transaction_id ASC", address, start_time, end_time)
payments = []
for tx in txs:
status = tx['status']
if status is None or status == 'queued':
status = 'unconfirmed'
value = parse_int(tx['value'])
if value is None:
value = 0
else:
value = hex(value)
# if the tx was created before the start time, send the unconfirmed
# message as well.
if status == 'confirmed' and tx['created'] > start_time:
payments.append(
SofaPayment(status='unconfirmed',
txHash=tx['hash'],
value=value,
fromAddress=tx['from_address'],
toAddress=tx['to_address'],
networkId=self.application.config['ethereum']
['network_id']).render())
payments.append(
SofaPayment(status=status,
txHash=tx['hash'],
value=value,
fromAddress=tx['from_address'],
toAddress=tx['to_address'],
networkId=self.application.config['ethereum']
['network_id']).render())
return payments
def database_transaction_to_rlp_transaction(transaction):
"""returns an rlp transaction for the given transaction"""
nonce = transaction['nonce']
value = parse_int(transaction['value'])
gas = parse_int(transaction['gas'])
gas_price = parse_int(transaction['gas_price'])
tx = create_transaction(nonce=nonce, gasprice=gas_price, startgas=gas,
to=transaction['to_address'], value=value,
data=data_decoder(transaction['data']),
v=parse_int(transaction['v']),
r=parse_int(transaction['r']),
s=parse_int(transaction['s']))
return tx
async def check_account_nonces(conf, dryrun=False):
async with conf.db.id.acquire() as con:
users = await con.fetch("SELECT toshi_id, payment_address FROM users")
last_percent = -1
async with conf.db.eth.acquire() as con:
tr = con.transaction()
await tr.start()
for i, user in enumerate(users):
percent = int((i / len(users)) * 100)
if percent != last_percent:
print("Progress: {}/{} ({}%)".format(i, len(users), percent))
last_percent = percent
from_address = user['payment_address']
resp = await app.http.post(
conf.urls.node,
headers={'Content-Type': 'application/json'},
data=json.dumps({
"jsonrpc": "2.0",
"id": random.randint(0, 1000000),
"method": "eth_getTransactionCount",
"params": [from_address]
}).encode('utf-8'))
if resp.status == 200:
data = await resp.json()
if 'result' in data and data['result'] is not None:
nonce = parse_int(data['result'])
res = await con.execute(
"UPDATE transactions SET last_status = 'error' WHERE from_address = $1 AND nonce >= $2 AND last_status != 'error'",
from_address, nonce)
if res != "UPDATE 0":
print("{}|{}: {}".format(user['toshi_id'],
from_address, res))
if dryrun:
await tr.rollback()
else:
await tr.commit()
async def create_transaction_skeleton(self,
*,
to_address,
from_address,
value=0,
nonce=None,
gas=None,
gas_price=None,
data=None):
if not validate_address(from_address):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_from_address',
'message': 'Invalid From Address'
})
if to_address is not None and not validate_address(to_address):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_to_address',
'message': 'Invalid To Address'
})
if value:
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_value',
'message': 'Invalid Value'
})
# check optional arguments
if nonce is None:
# check cache for nonce
nonce = await self.get_transaction_count(from_address)
else:
nonce = parse_int(nonce)
if nonce is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_nonce',
'message': 'Invalid Nonce'
})
if data is not None:
if isinstance(data, int):
data = hex(data)
if isinstance(data, str):
try:
data = data_decoder(data)
except binascii.Error:
pass
if not isinstance(data, bytes):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_data',
'message': 'Invalid Data field'
})
else:
data = b''
if gas is None:
try:
gas = await self.eth.eth_estimateGas(from_address,
to_address,
data=data,
value=value)
except JsonRPCError:
# this can occur if sending a transaction to a contract that doesn't match a valid method
# and the contract has no default method implemented
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_data',
'message': 'Invalid Data field'
})
else:
gas = parse_int(gas)
if gas is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_gas',
'message': 'Invalid Gas'
})
if gas_price is None:
gas_price = self.application.config['ethereum'].getint(
'default_gasprice', DEFAULT_GASPRICE)
else:
gas_price = parse_int(gas_price)
if gas_price is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_gas_price',
'message': 'Invalid Gas Price'
})
try:
tx = create_transaction(nonce=nonce,
gasprice=gas_price,
startgas=gas,
to=to_address,
value=value,
data=data,
network_id=self.network_id)
except InvalidTransaction as e:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': str(e)
})
if tx.intrinsic_gas_used > gas:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_transaction',
'message':
'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'
.format(tx.intrinsic_gas_used, gas)
})
transaction = encode_transaction(tx)
return transaction
def network_id(self):
return parse_int(self.application.config['ethereum']['network_id'])
async def get(self, address):
self.set_header("Access-Control-Allow-Origin", "*")
self.set_header("Access-Control-Allow-Headers", "x-requested-with")
self.set_header('Access-Control-Allow-Methods', 'GET')
offset = parse_int(self.get_argument('offset', '0'))
limit = parse_int(self.get_argument('limit', '10'))
status = set([s.lower() for s in self.get_arguments('status')])
direction = set([d.lower() for d in self.get_arguments('direction')])
order = self.get_argument('order', 'desc').upper()
if not validate_address(address) or \
offset is None or \
limit is None or \
(status and not status.issubset(['confirmed', 'unconfirmed', 'queued', 'error'])) or \
(direction and not direction.issubset(['in', 'out'])) or \
(order not in ['DESC', 'ASC']):
raise JSONHTTPError(400,
body={
'id': 'bad_arguments',
'message': 'Bad Arguments'
})
query = "SELECT * FROM transactions WHERE "
args = [address, offset, limit]
if len(direction) == 0 or len(direction) == 2:
query += "(from_address = $1 OR to_address = $1) "
elif 'in' in direction:
query += "to_address = $1 "
elif 'out' in direction:
query += "from_address = $1 "
if len(status) == 0:
query += "AND (status != $4 OR status IS NULL) "
args.append('error')
else:
status_query = []
for s in status:
if s == 'queued':
status_query.extend([
"status = ${}".format(len(args) + 1), "status IS NULL"
])
else:
status_query.append("status = ${}".format(len(args) + 1))
args.append(s)
query += "AND (" + " OR ".join(status_query) + ") "
query += "ORDER BY created {} OFFSET $2 LIMIT $3".format(order)
async with self.db:
rows = await self.db.fetch(query, *args)
transactions = []
for row in rows:
transactions.append({
"hash":
row['hash'],
"to":
row['to_address'],
"from":
row['from_address'],
"nonce":
hex(row['nonce']),
"value":
row['value'],
"gas":
row['gas'],
"gas_price":
row['gas_price'],
"created_data":
row['created'].isoformat(),
"confirmed_data":
row['updated'].isoformat() if row['blocknumber'] else None,
"status":
row['status'] if row['status'] is not None else 'queued',
"data":
row['data']
})
resp = {
"transactions": transactions,
"offset": offset,
"limit": limit,
"order": order
}
if len(direction) == 1:
resp['direction'] = direction.pop()
if status:
resp['status'] = "&".join(status)
self.write(resp)
async def get_apps(request, conf, current_user):
page = parse_int(request.args.get('page', None)) or 1
if page < 1:
page = 1
limit = 10
offset = (page - 1) * limit
order_by = request.args.get('order_by', None)
search_query = request.args.get('query', None)
filter_by = request.args.get('filter', None)
order = ('created', 'DESC')
if order_by:
if order_by in sortable_apps_columns:
if order_by[0] == '-':
order = (order_by[1:], 'ASC'
if order_by[1:] in negative_apps_columns else 'DESC')
else:
order = (order_by, 'DESC'
if order_by in negative_apps_columns else 'ASC')
if search_query:
# strip punctuation
query = ''.join(
[c for c in search_query if c not in string.punctuation])
# split words and add in partial matching flags
query = '|'.join(
['{}:*'.format(word) for word in query.split(' ') if word])
args = [offset, limit, query]
if order_by:
query_order = "ORDER BY {} {}".format(*order)
else:
# default order by rank
query_order = "ORDER BY TS_RANK_CD(t1.tsv, TO_TSQUERY($3)) DESC, name, username"
sql = ("SELECT * FROM "
"(SELECT * FROM users, TO_TSQUERY($3) AS q "
"WHERE (tsv @@ q) AND is_app = true) AS t1 "
"{} "
"OFFSET $1 LIMIT $2".format(query_order))
count_args = [query]
count_sql = ("SELECT COUNT(*) FROM users, TO_TSQUERY($1) AS q "
"WHERE (tsv @@ q) AND is_app = true")
async with conf.db.id.acquire() as con:
rows = await con.fetch(sql, *args)
count = await con.fetchrow(count_sql, *count_args)
else:
async with conf.db.id.acquire() as con:
rows = await con.fetch(
"SELECT * FROM users WHERE is_app = true ORDER BY {} {} NULLS LAST OFFSET $1 LIMIT $2"
.format(*order), offset, limit)
count = await con.fetchrow(
"SELECT COUNT(*) FROM users WHERE is_app = true")
apps = []
for row in rows:
app = fix_avatar_for_user(conf.urls.id, dict(row))
apps.append(app)
total_pages = (count['count'] // limit) + (0 if count['count'] %
limit == 0 else 1)
def get_qargs(page=page,
order_by=order_by,
query=search_query,
filter=filter_by,
as_list=False,
as_dict=False):
qargs = {'page': page}
if order_by:
if order_by[0] == '+':
order_by = order_by[1:]
elif order_by[0] != '-':
# toggle sort order
print(order, order_by)
if order[0] == order_by and order[1] == (
'ASC'
if order_by in negative_user_columns else 'DESC'):
order_by = '-{}'.format(order_by)
qargs['order_by'] = order_by
if query:
qargs['query'] = query
if filter:
qargs['filter'] = filter
if as_dict:
return qargs
if as_list:
return qargs.items()
return urlencode(qargs)
return html(await env.get_template("apps.html").render_async(
apps=apps,
current_user=current_user,
environment=conf.name,
page="apps",
total=count['count'],
total_pages=total_pages,
current_page=page,
get_qargs=get_qargs))
async def update_token_cache(self,
contract_address,
*eth_addresses,
blocknumber=None):
if len(eth_addresses) == 0:
return
is_wildcard = contract_address == "*"
async with self.db:
last_blocknumber = (
await
self.db.fetchval("SELECT blocknumber FROM last_blocknumber"))
if blocknumber is None:
blocknumber = last_blocknumber
elif blocknumber > last_blocknumber:
# don't continue until the block numbers match
log.info(
"request to update erc20 cache before block processor is caught up"
)
erc20_dispatcher.update_token_cache(
contract_address, *eth_addresses,
blocknumber=blocknumber).delay(1)
return
if is_wildcard:
tokens = await self.db.fetch(
"SELECT contract_address FROM tokens where custom = FALSE")
else:
tokens = [{'contract_address': contract_address}]
if is_wildcard:
if len(eth_addresses) > 1:
# this is currently unneeded and dangerous
raise Exception(
"wildcard update of token caches unsupported for multiple addresses"
)
log.info("START update_token_cache(\"*\", {})".format(
eth_addresses[0]))
start_time = time.time()
# NOTE: we don't remove this at the end on purpose
# to avoid spamming of "*" refreshes
should_run = await self.redis.set(
"bulk_token_update:{}".format(eth_addresses[0]),
1,
expire=60,
exist=self.redis.SET_IF_NOT_EXIST)
if not should_run:
log.info("ABORT update_token_cache(\"*\", {}): {}".format(
eth_addresses[0], should_run))
return
client = self.eth.bulk()
futures = []
for eth_address in eth_addresses:
for token in tokens:
data = "0x70a08231000000000000000000000000" + eth_address[2:]
f = client.eth_call(to_address=token['contract_address'],
data=data,
block=blocknumber)
futures.append((token['contract_address'], eth_address, f))
if len(futures) > 0:
await client.execute()
bulk_insert = []
for token_contract_address, eth_address, f in futures:
try:
value = f.result()
if value == "0x0000000000000000000000000000000000000000000000000000000000000000" or value == "0x":
if value == "0x":
log.warning(
"calling balanceOf for contract {} failed".
format(token_contract_address))
value = 0
else:
value = parse_int(value) # remove hex padding of value
bulk_insert.append(
(token_contract_address, eth_address, hex(value)))
except JsonRPCError as e:
if e.message == "Unknown Block Number":
# reschedule the update and abort for now
log.info(
"got unknown block number in erc20 cache update")
erc20_dispatcher.update_token_cache(
contract_address,
*eth_addresses,
blocknumber=blocknumber).delay(1)
return
log.exception(
"WARNING: failed to update token cache of '{}' for address: {}"
.format(token_contract_address, eth_address))
send_update = False
if len(bulk_insert) > 0:
async with self.db:
await self.db.executemany(
"INSERT INTO token_balances (contract_address, eth_address, balance) "
"VALUES ($1, $2, $3) "
"ON CONFLICT (contract_address, eth_address) "
"DO UPDATE set balance = EXCLUDED.balance",
bulk_insert)
await self.db.commit()
send_update = True
# wildcard updates usually mean we need to send a refresh trigger to clients
# currently clients only use a TokenPayment as a trigger to refresh their
# token cache, so we abuse this functionality here
if is_wildcard and send_update:
# lots of fake values so it doesn't get confused with a real tx
data = {
"txHash":
"0x0000000000000000000000000000000000000000000000000000000000000000",
"fromAddress":
"0x0000000000000000000000000000000000000000",
"toAddress": eth_addresses[0],
"status": "confirmed",
"value": "0x0",
"contractAddress":
"0x0000000000000000000000000000000000000000"
}
message = "SOFA::TokenPayment: " + json_encode(data)
manager_dispatcher.send_notification(eth_addresses[0], message)
if is_wildcard:
end_time = time.time()
log.info("DONE update_token_cache(\"*\", {}) in {}s".format(
eth_addresses[0], round(end_time - start_time, 2)))
async def liveordev(request, conf, user):
# get statistics
async with conf.db.eth.acquire() as con:
tx24h = await con.fetchrow(
"SELECT COUNT(*) FROM transactions WHERE created > (now() AT TIME ZONE 'utc') - interval '24 hours'"
)
tx7d = await con.fetchrow(
"SELECT COUNT(*) FROM transactions WHERE created > (now() AT TIME ZONE 'utc') - interval '7 days'"
)
tx1m = await con.fetchrow(
"SELECT COUNT(*) FROM transactions WHERE created > (now() AT TIME ZONE 'utc') - interval '1 month'"
)
txtotal = await con.fetchrow("SELECT COUNT(*) FROM transactions")
last_block = await con.fetchrow("SELECT * FROM last_blocknumber")
async with conf.db.id.acquire() as con:
u24h = await con.fetchrow(
"SELECT COUNT(*) FROM users WHERE created > (now() AT TIME ZONE 'utc') - interval '24 hours'"
)
u7d = await con.fetchrow(
"SELECT COUNT(*) FROM users WHERE created > (now() AT TIME ZONE 'utc') - interval '7 days'"
)
u1m = await con.fetchrow(
"SELECT COUNT(*) FROM users WHERE created > (now() AT TIME ZONE 'utc') - interval '1 month'"
)
utotal = await con.fetchrow("SELECT COUNT(*) FROM users")
users = {
'day': u24h['count'],
'week': u7d['count'],
'month': u1m['count'],
'total': utotal['count']
}
txs = {
'day': tx24h['count'],
'week': tx7d['count'],
'month': tx1m['count'],
'total': txtotal['count']
}
status = {}
block = {'db': last_block['blocknumber']}
# check service status
# eth
try:
resp = await app.http.get('{}/v1/balance/0x{}'.format(
conf.urls.eth, '0' * 40),
timeout=SERVICE_CHECK_TIMEOUT)
if resp.status == 200:
status['eth'] = "OK"
else:
status['eth'] = "Error: {}".format(resp.status)
except asyncio.TimeoutError:
status['eth'] = "Error: timeout"
# id
try:
resp = await app.http.get('{}/v1/user/0x{}'.format(
conf.urls.id, '0' * 40),
timeout=SERVICE_CHECK_TIMEOUT)
if resp.status == 404:
status['id'] = "OK"
else:
status['id'] = "Error: {}".format(resp.status)
except asyncio.TimeoutError:
status['id'] = "Error: timeout"
# dir
try:
resp = await app.http.get('{}/v1/apps/'.format(conf.urls.dir),
timeout=SERVICE_CHECK_TIMEOUT)
if resp.status == 200:
status['dir'] = "OK"
else:
status['dir'] = "Error: {}".format(resp.status)
except asyncio.TimeoutError:
status['dir'] = "Error: timeout"
# rep
try:
resp = await app.http.get('{}/v1/timestamp'.format(conf.urls.rep),
timeout=SERVICE_CHECK_TIMEOUT)
if resp.status == 200:
status['rep'] = "OK"
else:
status['rep'] = "Error: {}".format(resp.status)
except asyncio.TimeoutError:
status['rep'] = "Error: timeout"
# node
try:
resp = await app.http.post(
conf.urls.node,
headers={'Content-Type': 'application/json'},
data=json.dumps({
"jsonrpc": "2.0",
"id": random.randint(0, 1000000),
"method": "eth_blockNumber",
"params": []
}).encode('utf-8'))
if resp.status == 200:
data = await resp.json()
if 'result' in data:
if data['result'] is not None:
status['node'] = "OK"
block['node'] = parse_int(data['result'])
elif 'error' in data:
status['node'] = data['error']
else:
status['node'] = "Error: {}".format(resp.status)
except asyncio.TimeoutError:
status['node'] = "Error: timeout"
return html(await env.get_template("index.html").render_async(
current_user=user,
environment=conf.name,
page="home",
txs=txs,
users=users,
status=status,
block=block))
async def update_categories(request, conf, current_user):
category_name = request.form.get('category', None)
category_tag = request.form.get('tag', None)
category_id = request.form.get('id', None)
should_remove = request.form.get('remove', None)
language = 'en'
error = None
if should_remove is not None:
if category_id is None:
error = "Missing category id"
else:
category_id = parse_int(category_id)
async with conf.db.id.acquire() as con:
await con.fetchval(
"DELETE FROM categories WHERE category_id = $1",
category_id)
elif category_tag is None or category_name is None:
error = "Missing name and tag"
else:
# force tags to be lowercase
category_tag = category_tag.lower()
if category_id is None:
try:
async with conf.db.id.acquire() as con:
category_id = await con.fetchval(
"INSERT INTO categories (tag) VALUES ($1) RETURNING category_id",
category_tag)
except UniqueViolationError:
error = "Tag already exists"
# check again because we only update if the category_id was supplied by the user
# or the insert statement above succeeded.
if category_id is not None:
category_id = parse_int(category_id)
async with conf.db.id.acquire() as con:
await con.execute(
"INSERT INTO category_names (category_id, name, language) VALUES ($1, $2, $3) "
"ON CONFLICT (category_id, language) DO UPDATE SET name = EXCLUDED.name",
category_id, category_name, language)
get_sql = (
"SELECT * FROM categories "
"JOIN category_names ON categories.category_id = category_names.category_id AND category_names.language = $1"
"ORDER BY categories.tag ASC ")
async with conf.db.id.acquire() as con:
rows = await con.fetch(get_sql, language)
return html(await env.get_template("categories.html").render_async(
categories=rows,
error=error,
current_user=current_user,
environment=conf.name,
page="categories"))
def network_id(self):
return parse_int(config['ethereum']['network_id'])
async def create_transaction_skeleton(self, *, to_address, from_address, value=0, nonce=None, gas=None, gas_price=None, data=None):
if not validate_address(from_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address'})
if to_address is not None and not validate_address(to_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address'})
if from_address != from_address.lower() and not checksum_validate_address(from_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address Checksum'})
if to_address is not None and to_address != to_address.lower() and not checksum_validate_address(to_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address Checksum'})
if value:
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_value', 'message': 'Invalid Value'})
# check optional arguments
# check if we should ignore the given gasprice
# NOTE: only meant to be here while cryptokitty fever is pushing
# up gas prices... this shouldn't be perminant
# anytime the nonce is also set, use the provided gas (this is to
# support easier overwriting of transactions)
if gas_price is not None and nonce is None:
async with self.db:
whitelisted = await self.db.fetchrow("SELECT 1 FROM from_address_gas_price_whitelist WHERE address = $1", from_address)
if not whitelisted:
whitelisted = await self.db.fetchrow("SELECT 1 FROM to_address_gas_price_whitelist WHERE address = $1", to_address)
if not whitelisted:
gas_price = None
if nonce is None:
# check cache for nonce
nonce = await self.get_transaction_count(from_address)
else:
nonce = parse_int(nonce)
if nonce is None:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Invalid Nonce'})
if data is not None:
if isinstance(data, int):
data = hex(data)
if isinstance(data, str):
try:
data = data_decoder(data)
except binascii.Error:
pass
if not isinstance(data, bytes):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Invalid Data field'})
else:
data = b''
if gas is None:
try:
gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=value)
except JsonRPCError:
# this can occur if sending a transaction to a contract that doesn't match a valid method
# and the contract has no default method implemented
raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Unable to estimate gas for contract call'})
else:
gas = parse_int(gas)
if gas is None:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas', 'message': 'Invalid Gas'})
if gas_price is None:
# try and use cached gas station gas price
gas_station_gas_price = self.redis.get('gas_station_standard_gas_price')
if gas_station_gas_price:
gas_price = parse_int(gas_station_gas_price)
if gas_price is None:
gas_price = self.application.config['ethereum'].getint('default_gasprice', DEFAULT_GASPRICE)
else:
gas_price = parse_int(gas_price)
if gas_price is None:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas_price', 'message': 'Invalid Gas Price'})
try:
tx = create_transaction(nonce=nonce, gasprice=gas_price, startgas=gas,
to=to_address, value=value, data=data,
network_id=self.network_id)
except InvalidTransaction as e:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction', 'message': str(e)})
if tx.intrinsic_gas_used > gas:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'.format(
tx.intrinsic_gas_used, gas)})
transaction = encode_transaction(tx)
return {"tx": transaction, "gas": hex(gas), "gas_price": hex(gas_price), "nonce": hex(nonce), "value": hex(value)}
async def send_transaction(self, *, tx, signature=None):
try:
tx = decode_transaction(tx)
except:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction', 'message': 'Invalid Transaction'})
if is_transaction_signed(tx):
tx_sig = data_encoder(signature_from_transaction(tx))
if signature:
if tx_sig != signature:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature: Signature in payload and signature of transaction do not match'
})
else:
signature = tx_sig
else:
if signature is None:
raise JsonRPCInvalidParamsError(data={'id': 'missing_signature', 'message': 'Missing Signature'})
if not validate_signature(signature):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature: {}'.format(
'Invalid length' if len(signature) != 132 else 'Invalid hex value')
})
try:
sig = data_decoder(signature)
except Exception:
log.exception("Unexpected error decoding valid signature: {}".format(signature))
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature'
})
add_signature_to_transaction(tx, sig)
# validate network id, if it's not for "all networks"
if tx.network_id is not None and self.network_id != tx.network_id:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_network_id',
'message': 'Invalid Network ID'
})
from_address = data_encoder(tx.sender)
to_address = data_encoder(tx.to)
# prevent spamming of transactions with the same nonce from the same sender
with RedisLock(self.redis, "{}:{}".format(from_address, tx.nonce),
raise_when_locked=partial(JsonRPCInvalidParamsError, data={'id': 'invalid_nonce', 'message': 'Nonce already used'}),
ex=5):
# check for transaction overwriting
async with self.db:
existing = await self.db.fetchrow("SELECT * FROM transactions WHERE "
"from_address = $1 AND nonce = $2 AND status != $3",
from_address, tx.nonce, 'error')
# disallow transaction overwriting when the gas is lower or the transaction is confirmed
if existing and (parse_int(existing['gas_price']) >= tx.gasprice or existing['status'] == 'confirmed'):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Nonce already used'})
# make sure the account has enough funds for the transaction
network_balance, balance, _, _ = await self.get_balances(from_address)
if existing:
balance += parse_int(existing['value']) + parse_int(existing['gas']) * parse_int(existing['gas_price'])
if balance < (tx.value + (tx.startgas * tx.gasprice)):
raise JsonRPCInsufficientFundsError(data={'id': 'insufficient_funds', 'message': 'Insufficient Funds'})
# validate the nonce (only necessary if tx doesn't already exist)
if not existing:
c_nonce = await self.get_transaction_count(from_address)
if tx.nonce < c_nonce:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Provided nonce is too low'})
if tx.nonce > c_nonce:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Provided nonce is too high'})
if tx.intrinsic_gas_used > tx.startgas:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'.format(tx.intrinsic_gas_used, tx.startgas)})
# now this tx fits enough of the criteria to allow it
# onto the transaction queue
tx_hash = calculate_transaction_hash(tx)
if existing:
log.info("Setting tx '{}' to error due to forced overwrite".format(existing['hash']))
self.tasks.update_transaction(existing['transaction_id'], 'error')
# add tx to database
async with self.db:
await self.db.execute(
"INSERT INTO transactions "
"(hash, from_address, to_address, nonce, "
"value, gas, gas_price, "
"data, v, r, s, "
"sender_toshi_id) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
tx_hash, from_address, to_address, tx.nonce,
hex(tx.value), hex(tx.startgas), hex(tx.gasprice),
data_encoder(tx.data), hex(tx.v), hex(tx.r), hex(tx.s),
self.user_toshi_id)
await self.db.commit()
# trigger processing the transaction queue
self.tasks.process_transaction_queue(from_address)
# analytics
# use notification registrations to try find toshi ids for users
if self.user_toshi_id:
sender_toshi_id = self.user_toshi_id
else:
async with self.db:
sender_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1",
from_address)
async with self.db:
receiver_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1",
to_address)
self.track(sender_toshi_id, "Sent transaction")
# it doesn't make sense to add user agent here as we
# don't know the receiver's user agent
self.track(receiver_toshi_id, "Received transaction", add_user_agent=False)
return tx_hash
async def create_transaction_skeleton(self,
*,
to_address,
from_address,
value=0,
nonce=None,
gas=None,
gas_price=None,
data=None,
token_address=None):
if not validate_address(from_address):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_from_address',
'message': 'Invalid From Address'
})
if to_address is not None and not validate_address(to_address):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_to_address',
'message': 'Invalid To Address'
})
if from_address != from_address.lower(
) and not checksum_validate_address(from_address):
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_from_address',
'message': 'Invalid From Address Checksum'
})
if to_address is not None and to_address != to_address.lower(
) and not checksum_validate_address(to_address):
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_to_address',
'message': 'Invalid To Address Checksum'
})
# check if we should ignore the given gasprice
# NOTE: only meant to be here while cryptokitty fever is pushing
# up gas prices... this shouldn't be perminant
# anytime the nonce is also set, use the provided gas (this is to
# support easier overwriting of transactions)
if gas_price is not None and nonce is None:
async with self.db:
whitelisted = await self.db.fetchrow(
"SELECT 1 FROM from_address_gas_price_whitelist WHERE address = $1",
from_address)
if not whitelisted:
whitelisted = await self.db.fetchrow(
"SELECT 1 FROM to_address_gas_price_whitelist WHERE address = $1",
to_address)
if not whitelisted:
gas_price = None
if gas_price is None:
# try and use cached gas station gas price
gas_station_gas_price = await self.redis.get(
'gas_station_standard_gas_price')
if gas_station_gas_price:
gas_price = parse_int(gas_station_gas_price)
if gas_price is None:
gas_price = config['ethereum'].getint('default_gasprice',
DEFAULT_GASPRICE)
else:
gas_price = parse_int(gas_price)
if gas_price is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_gas_price',
'message': 'Invalid Gas Price'
})
if gas is not None:
gas = parse_int(gas)
if gas is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_gas',
'message': 'Invalid Gas'
})
if nonce is None:
# check cache for nonce
nonce = await self.get_transaction_count(from_address)
else:
nonce = parse_int(nonce)
if nonce is None:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_nonce',
'message': 'Invalid Nonce'
})
if data is not None:
if isinstance(data, int):
data = hex(data)
if isinstance(data, str):
try:
data = data_decoder(data)
except binascii.Error:
pass
if not isinstance(data, bytes):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_data',
'message': 'Invalid Data field'
})
else:
data = b''
# flag to force arguments into an erc20 token transfer
if token_address is not None:
if not validate_address(token_address):
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_token_address',
'message': 'Invalid Token Address'
})
if data != b'':
raise JsonRPCInvalidParamsError(
data={
'id': 'bad_arguments',
'message': 'Cannot include both data and token_address'
})
if isinstance(value, str) and value.lower() == "max":
# get the balance in the database
async with self.db:
value = await self.db.fetchval(
"SELECT value FROM token_balances "
"WHERE contract_address = $1 AND eth_address = $2",
token_address, from_address)
if value is None:
# get the value from the ethereum node
data = "0x70a08231000000000000000000000000" + from_address[
2:].lower()
value = await self.eth.eth_call(to_address=token_address,
data=data)
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_value',
'message': 'Invalid Value'
})
data = data_decoder(
"0xa9059cbb000000000000000000000000{}{:064x}".format(
to_address[2:].lower(), value))
token_value = value
value = 0
to_address = token_address
elif value:
if value == "max":
network_balance, balance, _, _ = await self.get_balances(
from_address)
if gas is None:
code = await self.eth.eth_getCode(to_address)
if code:
# we might have to do some work
try:
gas = await self.eth.eth_estimateGas(from_address,
to_address,
data=data,
value=0)
except JsonRPCError:
# no fallback function implemented in the contract means no ether can be sent to it
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_to_address',
'message':
'Cannot send payments to that address'
})
attempts = 0
# because the default function could do different things based on the eth sent, we make sure
# the value is suitable. if we get different values 3 times abort
while True:
if attempts > 2:
log.warning(
"Hit max attempts trying to get max value to send to contract '{}'"
.format(to_address))
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_to_address',
'message':
'Cannot send payments to that address'
})
value = balance - (gas_price * gas)
try:
gas_with_value = await self.eth.eth_estimateGas(
from_address,
to_address,
data=data,
value=value)
except JsonRPCError:
# no fallback function implemented in the contract means no ether can be sent to it
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_to_address',
'message':
'Cannot send payments to that address'
})
if gas_with_value != gas:
gas = gas_with_value
attempts += 1
continue
else:
break
else:
# normal address, 21000 gas per transaction
gas = 21000
value = balance - (gas_price * gas)
else:
# preset gas, run with it!
value = balance - (gas_price * gas)
else:
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_value',
'message': 'Invalid Value'
})
if gas is None:
try:
gas = await self.eth.eth_estimateGas(from_address,
to_address,
data=data,
value=value)
except JsonRPCError:
# this can occur if sending a transaction to a contract that doesn't match a valid method
# and the contract has no default method implemented.
# this can also happen if the current state of the blockchain means that submitting the
# transaction would fail (abort).
if token_address is not None:
# when dealing with erc20, this usually means the user's balance for that token isn't
# high enough, check that and throw an error if it's the case, and if not fall
# back to the standard invalid_data error
async with self.db:
bal = await self.db.fetchval(
"SELECT value FROM token_balances "
"WHERE contract_address = $1 AND eth_address = $2",
token_address, from_address)
if bal is not None:
bal = parse_int(bal)
if bal < token_value:
raise JsonRPCInsufficientFundsError(
data={
'id': 'insufficient_funds',
'message': 'Insufficient Funds'
})
raise JsonRPCInvalidParamsError(
data={
'id': 'invalid_data',
'message': 'Unable to estimate gas for contract call'
})
# if data is present, buffer gas estimate by 20%
if len(data) > 0:
gas = int(gas * 1.2)
try:
tx = create_transaction(nonce=nonce,
gasprice=gas_price,
startgas=gas,
to=to_address,
value=value,
data=data,
network_id=self.network_id)
except InvalidTransaction as e:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': str(e)
})
if tx.intrinsic_gas_used > gas:
raise JsonRPCInvalidParamsError(
data={
'id':
'invalid_transaction',
'message':
'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'
.format(tx.intrinsic_gas_used, gas)
})
transaction = encode_transaction(tx)
return {
"tx": transaction,
"gas": hex(gas),
"gas_price": hex(gas_price),
"nonce": hex(nonce),
"value": hex(token_value) if token_address else hex(value)
}