async def cancel_queued_transaction(self, tx_hash, signature):
if not validate_transaction_hash(tx_hash):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction_hash', 'message': 'Invalid Transaction Hash'})
if not validate_signature(signature):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_signature', 'message': 'Invalid Signature'})
async with self.db:
tx = await self.db.fetchrow("SELECT * FROM transactions WHERE hash = $1 AND (status != 'error' OR status IS NULL)",
tx_hash)
if tx is None:
raise JsonRPCError(None, -32000, "Transaction not found",
{'id': 'not_found', 'message': 'Transaction not found'})
elif tx['status'] != 'queued' and tx['status'] is not None:
raise JsonRPCError(None, -32000, "Transaction already sent to node",
{'id': 'invalid_transaction_status', 'message': 'Transaction already sent to node'})
message = "Cancel transaction " + tx_hash
if not personal_ecrecover(message, signature, tx['from_address']):
raise JsonRPCError(None, -32000, "Permission Denied",
{'id': 'permission_denied', 'message': 'Permission Denied'})
log.info("Setting tx '{}' to error due to user cancelation".format(tx['hash']))
self.tasks.update_transaction(tx['transaction_id'], 'error')
async def post_login(request):
token = request.json['auth_token']
url = '{}/v1/login/verify/{}'.format(ID_SERVICE_LOGIN_URL, token)
resp = await app.http.get(url)
if resp.status != 200:
raise SanicException("Login Failed", status_code=401)
user = await resp.json()
toshi_id = user['toshi_id']
session_id = generate_session_id()
async with app.pool.acquire() as con:
admin = await con.fetchrow("SELECT * FROM admins WHERE toshi_id = $1",
toshi_id)
if admin:
await con.execute(
"INSERT INTO sessions (session_id, toshi_id) VALUES ($1, $2)",
session_id, toshi_id)
if admin:
response = json_response(user)
response.cookies['session'] = session_id
#response.cookies['session']['secure'] = True
return response
else:
toshi_log.info("Invalid login from: {}".format(toshi_id))
raise SanicException("Login Failed", status_code=401)
async def wait_for_migration(con, poll_frequency=1):
"""finds the latest expected database version and only exits once the current
version in the database matches. Use for sub processes that depend on a main
process handling database migration"""
if not os.path.exists("sql/create_tables.sql"):
log.warning(
"Missing sql/create_tables.sql: cannot initialise database")
return
version = 0
while True:
version += 1
if not os.path.exists("sql/migrate_{:08}.sql".format(version)):
version -= 1
break
while True:
try:
row = await con.fetchrow(
"SELECT version_number FROM database_version LIMIT 1")
if version == row['version_number']:
break
except asyncpg.exceptions.UndefinedTableError:
# if this happens, it could just be the first time starting the app,
# just keep waiting
pass
log.info("waiting for database migration...".format(version))
# wait some time before checking again
await asyncio.sleep(poll_frequency)
# done!
log.info("got database version: {}".format(version))
return
async def _start(self):
if 'database' in config:
from toshi.database import prepare_database
await prepare_database()
if 'redis' in config:
from toshi.redis import prepare_redis
await prepare_redis()
self.listen(tornado.options.options.port, xheaders=True)
log.info("Starting HTTP Server on port: {}".format(
tornado.options.options.port))
async def send_transaction(self, *, tx, signature=None):
try:
tx = decode_transaction(tx)
except:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction', 'message': 'Invalid Transaction'})
if is_transaction_signed(tx):
tx_sig = data_encoder(signature_from_transaction(tx))
if signature:
if tx_sig != signature:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature: Signature in payload and signature of transaction do not match'
})
else:
signature = tx_sig
else:
if signature is None:
raise JsonRPCInvalidParamsError(data={'id': 'missing_signature', 'message': 'Missing Signature'})
if not validate_signature(signature):
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature: {}'.format(
'Invalid length' if len(signature) != 132 else 'Invalid hex value')
})
try:
sig = data_decoder(signature)
except Exception:
log.exception("Unexpected error decoding valid signature: {}".format(signature))
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_signature',
'message': 'Invalid Signature'
})
add_signature_to_transaction(tx, sig)
# validate network id, if it's not for "all networks"
if tx.network_id is not None and self.network_id != tx.network_id:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_network_id',
'message': 'Invalid Network ID'
})
from_address = data_encoder(tx.sender)
to_address = data_encoder(tx.to)
# prevent spamming of transactions with the same nonce from the same sender
with RedisLock(self.redis, "{}:{}".format(from_address, tx.nonce),
raise_when_locked=partial(JsonRPCInvalidParamsError, data={'id': 'invalid_nonce', 'message': 'Nonce already used'}),
ex=5):
# check for transaction overwriting
async with self.db:
existing = await self.db.fetchrow("SELECT * FROM transactions WHERE "
"from_address = $1 AND nonce = $2 AND status != $3",
from_address, tx.nonce, 'error')
# disallow transaction overwriting when the gas is lower or the transaction is confirmed
if existing and (parse_int(existing['gas_price']) >= tx.gasprice or existing['status'] == 'confirmed'):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Nonce already used'})
# make sure the account has enough funds for the transaction
network_balance, balance, _, _ = await self.get_balances(from_address)
if existing:
balance += parse_int(existing['value']) + parse_int(existing['gas']) * parse_int(existing['gas_price'])
if balance < (tx.value + (tx.startgas * tx.gasprice)):
raise JsonRPCInsufficientFundsError(data={'id': 'insufficient_funds', 'message': 'Insufficient Funds'})
# validate the nonce (only necessary if tx doesn't already exist)
if not existing:
c_nonce = await self.get_transaction_count(from_address)
if tx.nonce < c_nonce:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Provided nonce is too low'})
if tx.nonce > c_nonce:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Provided nonce is too high'})
if tx.intrinsic_gas_used > tx.startgas:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'.format(tx.intrinsic_gas_used, tx.startgas)})
# now this tx fits enough of the criteria to allow it
# onto the transaction queue
tx_hash = calculate_transaction_hash(tx)
if existing:
log.info("Setting tx '{}' to error due to forced overwrite".format(existing['hash']))
self.tasks.update_transaction(existing['transaction_id'], 'error')
# add tx to database
async with self.db:
await self.db.execute(
"INSERT INTO transactions "
"(hash, from_address, to_address, nonce, "
"value, gas, gas_price, "
"data, v, r, s, "
"sender_toshi_id) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
tx_hash, from_address, to_address, tx.nonce,
hex(tx.value), hex(tx.startgas), hex(tx.gasprice),
data_encoder(tx.data), hex(tx.v), hex(tx.r), hex(tx.s),
self.user_toshi_id)
await self.db.commit()
# trigger processing the transaction queue
self.tasks.process_transaction_queue(from_address)
# analytics
# use notification registrations to try find toshi ids for users
if self.user_toshi_id:
sender_toshi_id = self.user_toshi_id
else:
async with self.db:
sender_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1",
from_address)
async with self.db:
receiver_toshi_id = await self.db.fetchval(
"SELECT toshi_id FROM notification_registrations WHERE "
"eth_address = $1",
to_address)
self.track(sender_toshi_id, "Sent transaction")
# it doesn't make sense to add user agent here as we
# don't know the receiver's user agent
self.track(receiver_toshi_id, "Received transaction", add_user_agent=False)
return tx_hash
async def create_transaction_skeleton(self, *, to_address, from_address, value=0, nonce=None, gas=None, gas_price=None, data=None, network_id=None, token_address=None):
# strip begining and trailing whitespace from addresses
if to_address is not None and isinstance(to_address, str):
to_address = to_address.strip()
if from_address is not None and isinstance(from_address, str):
from_address = from_address.strip()
if token_address is not None and isinstance(token_address, str):
token_address = token_address.strip()
if not validate_address(from_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address'})
if to_address is not None and not validate_address(to_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address'})
if from_address != from_address.lower() and not checksum_validate_address(from_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address Checksum'})
if to_address is not None and to_address != to_address.lower() and not checksum_validate_address(to_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address Checksum'})
# make sure if the network id is set that it matches the current network id
# NOTE: this is only meant as a sanity check, to make sure clients abort
# early if they think they're creating a request on a different network
if network_id:
log.info("NETWORK ID USED: {} ({})".format(network_id, type(network_id)))
if type(network_id) == str:
try:
network_id = int(network_id)
except ValueError:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_network_id', 'message': 'Invalid Network Id'})
elif type(network_id) != int:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_network_id', 'message': 'Invalid Network Id'})
if network_id != self.network_id:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_network_id',
'message': 'Network ID does not match. expected: {}'.format(self.network_id)})
# check if we should ignore the given gasprice
# NOTE: only meant to be here while cryptokitty fever is pushing
# up gas prices... this shouldn't be perminant
# anytime the nonce is also set, use the provided gas (this is to
# support easier overwriting of transactions)
if gas_price is not None and nonce is None:
async with self.db:
whitelisted = await self.db.fetchrow("SELECT 1 FROM from_address_gas_price_whitelist WHERE address = $1", from_address)
if not whitelisted:
whitelisted = await self.db.fetchrow("SELECT 1 FROM to_address_gas_price_whitelist WHERE address = $1", to_address)
if not whitelisted:
gas_price = None
if gas_price is None:
# try and use cached gas station gas price
gas_station_gas_price = await self.redis.get('gas_station_fast_gas_price')
if gas_station_gas_price:
gas_price = parse_int(gas_station_gas_price)
if gas_price is None:
gas_price = await self.eth.eth_gasPrice()
if gas_price is None:
gas_price = config['ethereum'].getint('default_gasprice', DEFAULT_GASPRICE)
else:
gas_price = parse_int(gas_price)
if gas_price is None:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas_price', 'message': 'Invalid Gas Price'})
if gas is not None:
gas = parse_int(gas)
if gas is None:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas', 'message': 'Invalid Gas'})
if nonce is None:
# check cache for nonce
nonce = await self.get_transaction_count(from_address)
else:
nonce = parse_int(nonce)
if nonce is None:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Invalid Nonce'})
if data is not None:
if isinstance(data, int):
data = hex(data)
if isinstance(data, str):
try:
data = data_decoder(data)
except binascii.Error:
pass
if not isinstance(data, bytes):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Invalid Data field'})
else:
data = b''
# flag to force arguments into an erc20 token transfer
if token_address is not None:
if not validate_address(token_address):
raise JsonRPCInvalidParamsError(data={'id': 'invalid_token_address', 'message': 'Invalid Token Address'})
if data != b'':
raise JsonRPCInvalidParamsError(data={'id': 'bad_arguments', 'message': 'Cannot include both data and token_address'})
if isinstance(value, str) and value.lower() == "max":
# get the balance in the database
async with self.db:
value = await self.db.fetchval("SELECT balance FROM token_balances "
"WHERE contract_address = $1 AND eth_address = $2",
token_address, from_address)
if value is None:
# get the value from the ethereum node
data = "0x70a08231000000000000000000000000" + from_address[2:].lower()
try:
value = await self.eth.eth_call(to_address=token_address, data=data)
except:
log.exception("Unable to get balance for token {} for address {}".format(token_address, from_address))
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_value', 'message': 'Invalid Value'})
data = data_decoder("0xa9059cbb000000000000000000000000{}{:064x}".format(to_address[2:].lower(), value))
token_value = value
value = 0
to_address = token_address
elif value:
if value == "max":
network_balance, balance, _, _ = await self.get_balances(from_address)
if gas is None:
code = await self.eth.eth_getCode(to_address)
if code:
# we might have to do some work
try:
gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=0)
except JsonRPCError:
# no fallback function implemented in the contract means no ether can be sent to it
raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Cannot send payments to that address'})
attempts = 0
# because the default function could do different things based on the eth sent, we make sure
# the value is suitable. if we get different values 3 times abort
while True:
if attempts > 2:
log.warning("Hit max attempts trying to get max value to send to contract '{}'".format(to_address))
raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Cannot send payments to that address'})
value = balance - (gas_price * gas)
# make sure the balance isn't negative
if value < 0:
raise JsonRPCInsufficientFundsError(data={'id': 'insufficient_funds', 'message': 'Insufficient Funds'})
try:
gas_with_value = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=value)
except JsonRPCError:
# no fallback function implemented in the contract means no ether can be sent to it
raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Cannot send payments to that address'})
if gas_with_value != gas:
gas = gas_with_value
attempts += 1
continue
else:
break
else:
# normal address, 21000 gas per transaction
gas = 21000
value = balance - (gas_price * gas)
else:
# preset gas, run with it!
value = balance - (gas_price * gas)
else:
value = parse_int(value)
if value is None or value < 0:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_value', 'message': 'Invalid Value'})
if gas is None:
try:
gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=value)
except JsonRPCError:
# this can occur if sending a transaction to a contract that doesn't match a valid method
# and the contract has no default method implemented.
# this can also happen if the current state of the blockchain means that submitting the
# transaction would fail (abort).
if token_address is not None:
# when dealing with erc20, this usually means the user's balance for that token isn't
# high enough, check that and throw an error if it's the case, and if not fall
# back to the standard invalid_data error
async with self.db:
bal = await self.db.fetchval("SELECT balance FROM token_balances "
"WHERE contract_address = $1 AND eth_address = $2",
token_address, from_address)
if bal is not None:
bal = parse_int(bal)
if bal < token_value:
raise JsonRPCInsufficientFundsError(data={'id': 'insufficient_funds', 'message': 'Insufficient Funds'})
raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Unable to estimate gas for contract call'})
# if data is present, buffer gas estimate by 20%
if len(data) > 0:
gas = int(gas * 1.2)
try:
tx = create_transaction(nonce=nonce, gasprice=gas_price, startgas=gas,
to=to_address, value=value, data=data,
network_id=self.network_id)
except InvalidTransaction as e:
raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction', 'message': str(e)})
except binascii.Error as e:
log.exception("Error creating transaction skeleton: nonce:{} gasprice:{} startgas:{} to:{} value:{} data:{} network_id:{}".format(
nonce, gas_price, gas, to_address, value, data, self.network_id))
raise JsonRPCError(None, -32000, "Error creating transaction skeleton",
{'id': 'unexpected_error', 'message': "Error creating transaction skeleton"})
if tx.intrinsic_gas_used > gas:
raise JsonRPCInvalidParamsError(data={
'id': 'invalid_transaction',
'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'.format(
tx.intrinsic_gas_used, gas)})
transaction = encode_transaction(tx)
return {"tx": transaction, "gas": hex(gas), "gas_price": hex(gas_price), "nonce": hex(nonce),
"value": hex(token_value) if token_address else hex(value)}
async def create_tables(con):
# make sure the create tables script exists
if not os.path.exists("sql/create_tables.sql"):
log.warning(
"Missing sql/create_tables.sql: cannot initialise database")
return
try:
row = await con.fetchrow(
"SELECT version_number FROM database_version LIMIT 1")
version = row['version_number']
log.info("got database version: {}".format(version))
except asyncpg.exceptions.UndefinedTableError:
# fresh DB path
await con.execute(
"CREATE TABLE database_version (version_number INTEGER)")
await con.execute(
"INSERT INTO database_version (version_number) VALUES (0)")
# fresh database, nothing to migrate
with open("sql/create_tables.sql") as create_tables_file:
sql = create_tables_file.read()
await con.execute(sql)
# verify that if there are any migration scripts, that the
# database_version table has been updated appropriately
version = 0
while True:
version += 1
if not os.path.exists("sql/migrate_{:08}.sql".format(version)):
version -= 1
break
if version > 0:
row = await con.fetchrow(
"SELECT version_number FROM database_version LIMIT 1")
if row['version_number'] != version:
log.warning(
"Warning, migration scripts exist but database version has not been set in create_tables.sql"
)
log.warning(
"DB version: {}, latest migration script: {}".format(
row['version_number'], version))
return
# check for migration files
exception = None
while True:
version += 1
fn = "sql/migrate_{:08}.sql".format(version)
if os.path.exists(fn):
log.info("applying migration script: {:08}".format(version))
with open(fn) as migrate_file:
sql = migrate_file.read()
try:
await con.execute(sql)
except Exception as e:
version -= 1
exception = e
break
else:
version -= 1
break
await con.execute("UPDATE database_version SET version_number = $1",
version)
if exception:
raise exception