async def cancel_queued_transaction(self, tx_hash, signature): if not validate_transaction_hash(tx_hash): raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction_hash', 'message': 'Invalid Transaction Hash'}) if not validate_signature(signature): raise JsonRPCInvalidParamsError(data={'id': 'invalid_signature', 'message': 'Invalid Signature'}) async with self.db: tx = await self.db.fetchrow("SELECT * FROM transactions WHERE hash = $1 AND (status != 'error' OR status IS NULL)", tx_hash) if tx is None: raise JsonRPCError(None, -32000, "Transaction not found", {'id': 'not_found', 'message': 'Transaction not found'}) elif tx['status'] != 'queued' and tx['status'] is not None: raise JsonRPCError(None, -32000, "Transaction already sent to node", {'id': 'invalid_transaction_status', 'message': 'Transaction already sent to node'}) message = "Cancel transaction " + tx_hash if not personal_ecrecover(message, signature, tx['from_address']): raise JsonRPCError(None, -32000, "Permission Denied", {'id': 'permission_denied', 'message': 'Permission Denied'}) log.info("Setting tx '{}' to error due to user cancelation".format(tx['hash'])) self.tasks.update_transaction(tx['transaction_id'], 'error')
async def post_login(request): token = request.json['auth_token'] url = '{}/v1/login/verify/{}'.format(ID_SERVICE_LOGIN_URL, token) resp = await app.http.get(url) if resp.status != 200: raise SanicException("Login Failed", status_code=401) user = await resp.json() toshi_id = user['toshi_id'] session_id = generate_session_id() async with app.pool.acquire() as con: admin = await con.fetchrow("SELECT * FROM admins WHERE toshi_id = $1", toshi_id) if admin: await con.execute( "INSERT INTO sessions (session_id, toshi_id) VALUES ($1, $2)", session_id, toshi_id) if admin: response = json_response(user) response.cookies['session'] = session_id #response.cookies['session']['secure'] = True return response else: toshi_log.info("Invalid login from: {}".format(toshi_id)) raise SanicException("Login Failed", status_code=401)
async def wait_for_migration(con, poll_frequency=1): """finds the latest expected database version and only exits once the current version in the database matches. Use for sub processes that depend on a main process handling database migration""" if not os.path.exists("sql/create_tables.sql"): log.warning( "Missing sql/create_tables.sql: cannot initialise database") return version = 0 while True: version += 1 if not os.path.exists("sql/migrate_{:08}.sql".format(version)): version -= 1 break while True: try: row = await con.fetchrow( "SELECT version_number FROM database_version LIMIT 1") if version == row['version_number']: break except asyncpg.exceptions.UndefinedTableError: # if this happens, it could just be the first time starting the app, # just keep waiting pass log.info("waiting for database migration...".format(version)) # wait some time before checking again await asyncio.sleep(poll_frequency) # done! log.info("got database version: {}".format(version)) return
async def _start(self): if 'database' in config: from toshi.database import prepare_database await prepare_database() if 'redis' in config: from toshi.redis import prepare_redis await prepare_redis() self.listen(tornado.options.options.port, xheaders=True) log.info("Starting HTTP Server on port: {}".format( tornado.options.options.port))
async def send_transaction(self, *, tx, signature=None): try: tx = decode_transaction(tx) except: raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction', 'message': 'Invalid Transaction'}) if is_transaction_signed(tx): tx_sig = data_encoder(signature_from_transaction(tx)) if signature: if tx_sig != signature: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_signature', 'message': 'Invalid Signature: Signature in payload and signature of transaction do not match' }) else: signature = tx_sig else: if signature is None: raise JsonRPCInvalidParamsError(data={'id': 'missing_signature', 'message': 'Missing Signature'}) if not validate_signature(signature): raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_signature', 'message': 'Invalid Signature: {}'.format( 'Invalid length' if len(signature) != 132 else 'Invalid hex value') }) try: sig = data_decoder(signature) except Exception: log.exception("Unexpected error decoding valid signature: {}".format(signature)) raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_signature', 'message': 'Invalid Signature' }) add_signature_to_transaction(tx, sig) # validate network id, if it's not for "all networks" if tx.network_id is not None and self.network_id != tx.network_id: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_network_id', 'message': 'Invalid Network ID' }) from_address = data_encoder(tx.sender) to_address = data_encoder(tx.to) # prevent spamming of transactions with the same nonce from the same sender with RedisLock(self.redis, "{}:{}".format(from_address, tx.nonce), raise_when_locked=partial(JsonRPCInvalidParamsError, data={'id': 'invalid_nonce', 'message': 'Nonce already used'}), ex=5): # check for transaction overwriting async with self.db: existing = await self.db.fetchrow("SELECT * FROM transactions WHERE " "from_address = $1 AND nonce = $2 AND status != $3", from_address, tx.nonce, 'error') # disallow transaction overwriting when the gas is lower or the transaction is confirmed if existing and (parse_int(existing['gas_price']) >= tx.gasprice or existing['status'] == 'confirmed'): raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Nonce already used'}) # make sure the account has enough funds for the transaction network_balance, balance, _, _ = await self.get_balances(from_address) if existing: balance += parse_int(existing['value']) + parse_int(existing['gas']) * parse_int(existing['gas_price']) if balance < (tx.value + (tx.startgas * tx.gasprice)): raise JsonRPCInsufficientFundsError(data={'id': 'insufficient_funds', 'message': 'Insufficient Funds'}) # validate the nonce (only necessary if tx doesn't already exist) if not existing: c_nonce = await self.get_transaction_count(from_address) if tx.nonce < c_nonce: raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Provided nonce is too low'}) if tx.nonce > c_nonce: raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Provided nonce is too high'}) if tx.intrinsic_gas_used > tx.startgas: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_transaction', 'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'.format(tx.intrinsic_gas_used, tx.startgas)}) # now this tx fits enough of the criteria to allow it # onto the transaction queue tx_hash = calculate_transaction_hash(tx) if existing: log.info("Setting tx '{}' to error due to forced overwrite".format(existing['hash'])) self.tasks.update_transaction(existing['transaction_id'], 'error') # add tx to database async with self.db: await self.db.execute( "INSERT INTO transactions " "(hash, from_address, to_address, nonce, " "value, gas, gas_price, " "data, v, r, s, " "sender_toshi_id) " "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", tx_hash, from_address, to_address, tx.nonce, hex(tx.value), hex(tx.startgas), hex(tx.gasprice), data_encoder(tx.data), hex(tx.v), hex(tx.r), hex(tx.s), self.user_toshi_id) await self.db.commit() # trigger processing the transaction queue self.tasks.process_transaction_queue(from_address) # analytics # use notification registrations to try find toshi ids for users if self.user_toshi_id: sender_toshi_id = self.user_toshi_id else: async with self.db: sender_toshi_id = await self.db.fetchval( "SELECT toshi_id FROM notification_registrations WHERE " "eth_address = $1", from_address) async with self.db: receiver_toshi_id = await self.db.fetchval( "SELECT toshi_id FROM notification_registrations WHERE " "eth_address = $1", to_address) self.track(sender_toshi_id, "Sent transaction") # it doesn't make sense to add user agent here as we # don't know the receiver's user agent self.track(receiver_toshi_id, "Received transaction", add_user_agent=False) return tx_hash
async def create_transaction_skeleton(self, *, to_address, from_address, value=0, nonce=None, gas=None, gas_price=None, data=None, network_id=None, token_address=None): # strip begining and trailing whitespace from addresses if to_address is not None and isinstance(to_address, str): to_address = to_address.strip() if from_address is not None and isinstance(from_address, str): from_address = from_address.strip() if token_address is not None and isinstance(token_address, str): token_address = token_address.strip() if not validate_address(from_address): raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address'}) if to_address is not None and not validate_address(to_address): raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address'}) if from_address != from_address.lower() and not checksum_validate_address(from_address): raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address Checksum'}) if to_address is not None and to_address != to_address.lower() and not checksum_validate_address(to_address): raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address Checksum'}) # make sure if the network id is set that it matches the current network id # NOTE: this is only meant as a sanity check, to make sure clients abort # early if they think they're creating a request on a different network if network_id: log.info("NETWORK ID USED: {} ({})".format(network_id, type(network_id))) if type(network_id) == str: try: network_id = int(network_id) except ValueError: raise JsonRPCInvalidParamsError(data={'id': 'invalid_network_id', 'message': 'Invalid Network Id'}) elif type(network_id) != int: raise JsonRPCInvalidParamsError(data={'id': 'invalid_network_id', 'message': 'Invalid Network Id'}) if network_id != self.network_id: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_network_id', 'message': 'Network ID does not match. expected: {}'.format(self.network_id)}) # check if we should ignore the given gasprice # NOTE: only meant to be here while cryptokitty fever is pushing # up gas prices... this shouldn't be perminant # anytime the nonce is also set, use the provided gas (this is to # support easier overwriting of transactions) if gas_price is not None and nonce is None: async with self.db: whitelisted = await self.db.fetchrow("SELECT 1 FROM from_address_gas_price_whitelist WHERE address = $1", from_address) if not whitelisted: whitelisted = await self.db.fetchrow("SELECT 1 FROM to_address_gas_price_whitelist WHERE address = $1", to_address) if not whitelisted: gas_price = None if gas_price is None: # try and use cached gas station gas price gas_station_gas_price = await self.redis.get('gas_station_fast_gas_price') if gas_station_gas_price: gas_price = parse_int(gas_station_gas_price) if gas_price is None: gas_price = await self.eth.eth_gasPrice() if gas_price is None: gas_price = config['ethereum'].getint('default_gasprice', DEFAULT_GASPRICE) else: gas_price = parse_int(gas_price) if gas_price is None: raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas_price', 'message': 'Invalid Gas Price'}) if gas is not None: gas = parse_int(gas) if gas is None: raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas', 'message': 'Invalid Gas'}) if nonce is None: # check cache for nonce nonce = await self.get_transaction_count(from_address) else: nonce = parse_int(nonce) if nonce is None: raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Invalid Nonce'}) if data is not None: if isinstance(data, int): data = hex(data) if isinstance(data, str): try: data = data_decoder(data) except binascii.Error: pass if not isinstance(data, bytes): raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Invalid Data field'}) else: data = b'' # flag to force arguments into an erc20 token transfer if token_address is not None: if not validate_address(token_address): raise JsonRPCInvalidParamsError(data={'id': 'invalid_token_address', 'message': 'Invalid Token Address'}) if data != b'': raise JsonRPCInvalidParamsError(data={'id': 'bad_arguments', 'message': 'Cannot include both data and token_address'}) if isinstance(value, str) and value.lower() == "max": # get the balance in the database async with self.db: value = await self.db.fetchval("SELECT balance FROM token_balances " "WHERE contract_address = $1 AND eth_address = $2", token_address, from_address) if value is None: # get the value from the ethereum node data = "0x70a08231000000000000000000000000" + from_address[2:].lower() try: value = await self.eth.eth_call(to_address=token_address, data=data) except: log.exception("Unable to get balance for token {} for address {}".format(token_address, from_address)) value = parse_int(value) if value is None or value < 0: raise JsonRPCInvalidParamsError(data={'id': 'invalid_value', 'message': 'Invalid Value'}) data = data_decoder("0xa9059cbb000000000000000000000000{}{:064x}".format(to_address[2:].lower(), value)) token_value = value value = 0 to_address = token_address elif value: if value == "max": network_balance, balance, _, _ = await self.get_balances(from_address) if gas is None: code = await self.eth.eth_getCode(to_address) if code: # we might have to do some work try: gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=0) except JsonRPCError: # no fallback function implemented in the contract means no ether can be sent to it raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Cannot send payments to that address'}) attempts = 0 # because the default function could do different things based on the eth sent, we make sure # the value is suitable. if we get different values 3 times abort while True: if attempts > 2: log.warning("Hit max attempts trying to get max value to send to contract '{}'".format(to_address)) raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Cannot send payments to that address'}) value = balance - (gas_price * gas) # make sure the balance isn't negative if value < 0: raise JsonRPCInsufficientFundsError(data={'id': 'insufficient_funds', 'message': 'Insufficient Funds'}) try: gas_with_value = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=value) except JsonRPCError: # no fallback function implemented in the contract means no ether can be sent to it raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Cannot send payments to that address'}) if gas_with_value != gas: gas = gas_with_value attempts += 1 continue else: break else: # normal address, 21000 gas per transaction gas = 21000 value = balance - (gas_price * gas) else: # preset gas, run with it! value = balance - (gas_price * gas) else: value = parse_int(value) if value is None or value < 0: raise JsonRPCInvalidParamsError(data={'id': 'invalid_value', 'message': 'Invalid Value'}) if gas is None: try: gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=value) except JsonRPCError: # this can occur if sending a transaction to a contract that doesn't match a valid method # and the contract has no default method implemented. # this can also happen if the current state of the blockchain means that submitting the # transaction would fail (abort). if token_address is not None: # when dealing with erc20, this usually means the user's balance for that token isn't # high enough, check that and throw an error if it's the case, and if not fall # back to the standard invalid_data error async with self.db: bal = await self.db.fetchval("SELECT balance FROM token_balances " "WHERE contract_address = $1 AND eth_address = $2", token_address, from_address) if bal is not None: bal = parse_int(bal) if bal < token_value: raise JsonRPCInsufficientFundsError(data={'id': 'insufficient_funds', 'message': 'Insufficient Funds'}) raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Unable to estimate gas for contract call'}) # if data is present, buffer gas estimate by 20% if len(data) > 0: gas = int(gas * 1.2) try: tx = create_transaction(nonce=nonce, gasprice=gas_price, startgas=gas, to=to_address, value=value, data=data, network_id=self.network_id) except InvalidTransaction as e: raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction', 'message': str(e)}) except binascii.Error as e: log.exception("Error creating transaction skeleton: nonce:{} gasprice:{} startgas:{} to:{} value:{} data:{} network_id:{}".format( nonce, gas_price, gas, to_address, value, data, self.network_id)) raise JsonRPCError(None, -32000, "Error creating transaction skeleton", {'id': 'unexpected_error', 'message': "Error creating transaction skeleton"}) if tx.intrinsic_gas_used > gas: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_transaction', 'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'.format( tx.intrinsic_gas_used, gas)}) transaction = encode_transaction(tx) return {"tx": transaction, "gas": hex(gas), "gas_price": hex(gas_price), "nonce": hex(nonce), "value": hex(token_value) if token_address else hex(value)}
async def create_tables(con): # make sure the create tables script exists if not os.path.exists("sql/create_tables.sql"): log.warning( "Missing sql/create_tables.sql: cannot initialise database") return try: row = await con.fetchrow( "SELECT version_number FROM database_version LIMIT 1") version = row['version_number'] log.info("got database version: {}".format(version)) except asyncpg.exceptions.UndefinedTableError: # fresh DB path await con.execute( "CREATE TABLE database_version (version_number INTEGER)") await con.execute( "INSERT INTO database_version (version_number) VALUES (0)") # fresh database, nothing to migrate with open("sql/create_tables.sql") as create_tables_file: sql = create_tables_file.read() await con.execute(sql) # verify that if there are any migration scripts, that the # database_version table has been updated appropriately version = 0 while True: version += 1 if not os.path.exists("sql/migrate_{:08}.sql".format(version)): version -= 1 break if version > 0: row = await con.fetchrow( "SELECT version_number FROM database_version LIMIT 1") if row['version_number'] != version: log.warning( "Warning, migration scripts exist but database version has not been set in create_tables.sql" ) log.warning( "DB version: {}, latest migration script: {}".format( row['version_number'], version)) return # check for migration files exception = None while True: version += 1 fn = "sql/migrate_{:08}.sql".format(version) if os.path.exists(fn): log.info("applying migration script: {:08}".format(version)) with open(fn) as migrate_file: sql = migrate_file.read() try: await con.execute(sql) except Exception as e: version -= 1 exception = e break else: version -= 1 break await con.execute("UPDATE database_version SET version_number = $1", version) if exception: raise exception