def test_add_signature_to_transaction_with_netowrk_id(self): for network_id in [1, 2, 66, 100]: sender_private_key = "0x0164f7c7399f4bb1eafeaae699ebbb12050bc6a50b2836b9ca766068a9d000c0" sender_address = "0xde3d2d9dd52ea80f7799ef4791063a5458d13913" to_address = "0x056db290f8ba3250ca64a45d16284d04bc6f5fbf" value = 10000000000 nonce = 1048576 data = b'' gasprice = DEFAULT_GASPRICE startgas = DEFAULT_STARTGAS network_id = 1 tx1 = Transaction(nonce, gasprice, startgas, to_address, value, data, network_id, 0, 0) tx = encode_transaction(tx1) tx1.sign(data_decoder(sender_private_key), network_id=network_id) expected_signed_tx = encode_transaction(tx1) sig = data_encoder(signature_from_transaction(tx1)) signed_tx = add_signature_to_transaction(tx, sig) self.assertEqual(signed_tx, expected_signed_tx) tx_obj = decode_transaction(tx) add_signature_to_transaction(tx_obj, sig) self.assertEqual(tx_obj.network_id, network_id) self.assertEqual(data_encoder(tx_obj.sender), sender_address) self.assertEqual(encode_transaction(tx_obj), expected_signed_tx)
async def test_create_and_send_signed_transaction_with_separate_sig(self): body = {"from": FAUCET_ADDRESS, "to": TEST_ADDRESS, "value": 10**10} resp = await self.fetch("/tx/skel", method="POST", body=body) self.assertEqual(resp.code, 200) body = json_decode(resp.body) tx = decode_transaction(body['tx']) tx = sign_transaction(tx, FAUCET_PRIVATE_KEY) sig = signature_from_transaction(tx) body = {"tx": encode_transaction(tx), "signature": data_encoder(sig)} resp = await self.fetch("/tx", method="POST", body=body) self.assertEqual(resp.code, 200, resp.body) body = json_decode(resp.body) tx_hash = body['tx_hash'] async def check_db(): async with self.pool.acquire() as con: rows = await con.fetch( "SELECT * FROM transactions WHERE nonce = $1", tx.nonce) self.assertEqual(len(rows), 1) self.assertEqual(rows[0]['hash'], tx_hash) await self.wait_on_tx_confirmation(tx_hash)
async def test_transactions_get_queued_in_queue_is_weird(self, *, monitor, manager): """This test reproduces a bug where if there was a transaction with too low a gas price queued, the monitor sanity check would force any further queued transactions with a good gas price and cause the previous transaction to the error state due to the nonce being problematic. This tests that even if the queue gets in a weird state like this it still gets resolved correctly """ gas_price = 50000000000 safe_low_gas_price = 40000000000 too_low_gas_price = 30000000000 assert(gas_price != DEFAULT_GASPRICE) assert(gas_price > safe_low_gas_price) assert(safe_low_gas_price > too_low_gas_price) await self.redis.set("gas_station_standard_gas_price", hex(gas_price)) await self.redis.set("gas_station_safelow_gas_price", hex(safe_low_gas_price)) for nonce, gasprice, value in [(0, too_low_gas_price, 0x1000000000000), (1, safe_low_gas_price, 0x2000000000000), (2, gas_price, 0x3000000000000)]: tx = create_transaction(nonce=0x100000 + nonce, gasprice=gasprice, startgas=DEFAULT_STARTGAS, to=TEST_ADDRESS, value=value, network_id=0x42) await self.sign_and_send_tx(FAUCET_PRIVATE_KEY, encode_transaction(tx)) await asyncio.sleep(0.1) async with self.pool.acquire() as con: txs = await con.fetch("SELECT * FROM transactions") for tx in txs: self.assertEqual(tx['status'], 'queued') # purposefully cause some problems async with self.pool.acquire() as con: txs = await con.fetch("UPDATE transactions SET status = 'unconfirmed' WHERE nonce > $1", 0x100000) from toshieth.tasks import manager_dispatcher manager_dispatcher.process_transaction_queue(FAUCET_ADDRESS) await asyncio.sleep(0.1) async with self.pool.acquire() as con: tx = await con.fetchrow("SELECT * FROM transactions WHERE nonce = $1", 0x100000) self.assertEqual(tx['status'], 'queued') # fix up gas prices await self.redis.set("gas_station_standard_gas_price", hex(safe_low_gas_price)) await self.redis.set("gas_station_safelow_gas_price", hex(too_low_gas_price)) manager_dispatcher.process_transaction_queue(FAUCET_ADDRESS) await self.wait_on_tx_confirmation(tx['hash']) async with self.pool.acquire() as con: tx = await con.fetchrow("SELECT * FROM transactions WHERE nonce = $1", 0x100000) self.assertEqual(tx['status'], 'confirmed')
def send_tx(self, tx, signature=None, **kwargs): if isinstance(tx, (Transaction, UnsignedTransaction)): tx = encode_transaction(tx) body = {"tx": tx} if signature: body['signature'] = signature resp = self._fetch("/v1/tx", "POST", body, **kwargs) return resp['tx_hash']
async def test_get_balance(self, *, ethminer): addr = '0x39bf9e501e61440b4b268d7b2e9aa2458dd201bb' val = 8761751855997712 val2 = int(val / 2) await self.wait_on_tx_confirmation(await self.faucet(TEST_ID_ADDRESS, val)) ws_con = await self.websocket_connect(TEST_ID_KEY) result = await ws_con.call("get_balance", [TEST_ID_ADDRESS]) self.assertEqual(int(result['confirmed_balance'][2:], 16), val) self.assertEqual(int(result['unconfirmed_balance'][2:], 16), val) # make sure no blocks get mined for a bit ethminer.pause() tx = create_transaction(nonce=0x100000, gasprice=DEFAULT_GASPRICE, startgas=DEFAULT_STARTGAS, to=addr, value=val2, network_id=self.network_id) tx = sign_transaction(tx, TEST_ID_KEY) tx = encode_transaction(tx) await ws_con.call("subscribe", [addr]) tx_hash = await ws_con.call("send_transaction", {"tx": tx}) new_balance = val - (val2 + DEFAULT_STARTGAS * DEFAULT_GASPRICE) result = await ws_con.call("get_balance", [TEST_ID_ADDRESS]) self.assertEqual(int(result['confirmed_balance'][2:], 16), val) self.assertEqual(int(result['unconfirmed_balance'][2:], 16), new_balance) # check for the unconfirmed notification result = await ws_con.read() self.assertIsNotNone(result) payment = parse_sofa_message(result['params']['message']) self.assertEqual(payment['txHash'], tx_hash) self.assertEqual(payment['status'], 'unconfirmed') # restart mining ethminer.start() result = await ws_con.read() payment = parse_sofa_message(result['params']['message']) self.assertEqual(payment['txHash'], tx_hash) self.assertEqual(payment['status'], 'confirmed') result = await ws_con.call("get_balance", [TEST_ID_ADDRESS]) self.assertEqual(int(result['confirmed_balance'][2:], 16), new_balance, "int('{}', 16) != {}".format(result['confirmed_balance'], new_balance)) self.assertEqual(int(result['unconfirmed_balance'][2:], 16), new_balance, "int('{}', 16) != {}".format(result['unconfirmed_balance'], new_balance))
def test_get_skel(self): tx = self.service_client.generate_tx_skel( "0x0004DE837Ea93edbE51c093f45212AB22b4B35fc", "0xdb089a4f9a8c5f17040b4fc51647e942b5fc601d", 1000000000000000000, timeout=5) self.assertEqual( encode_transaction(tx), "0xe9808504a817c80082520894db089a4f9a8c5f17040b4fc51647e942b5fc601d880de0b6b3a764000080" )
async def test_send_txs_for_all_networks(self): tx = create_transaction(nonce=1048576, gasprice=20 * 10**9, startgas=21000, to="0x3535353535353535353535353535353535353535", value=10**18, data=b'') sign_transaction(tx, FAUCET_PRIVATE_KEY) resp = await self.fetch("/tx", method="POST", body={ "tx": encode_transaction(tx) }) self.assertEqual(resp.code, 200, resp.body) await self.wait_on_tx_confirmation(data_encoder(tx.hash))
async def test_unable_to_send_txs_for_other_networks(self): network_id = 1 # Test network id is 66 tx = create_transaction(nonce=9, gasprice=20 * 10**9, startgas=21000, to="0x3535353535353535353535353535353535353535", value=10**18, data=b'', network_id=network_id) sign_transaction(tx, FAUCET_PRIVATE_KEY) resp = await self.fetch("/tx", method="POST", body={ "tx": encode_transaction(tx) }) self.assertEqual(resp.code, 400, resp.body)
def test_add_signature_to_transaction(self): tx = "0xe9808504a817c80082520894db089a4f9a8c5f17040b4fc51647e942b5fc601d880de0b6b3a764000080" sig = "0xf5a43adea07d366ae420a5c75a5cae6c60d3e4aaa0b72c2f37fc387efd43d7fd30c4327f2dbd959f654857f58912129b09763329459d08e25547d895ae90fa0f01" expected_signed_tx = "0xf86c808504a817c80082520894db089a4f9a8c5f17040b4fc51647e942b5fc601d880de0b6b3a7640000801ca0f5a43adea07d366ae420a5c75a5cae6c60d3e4aaa0b72c2f37fc387efd43d7fda030c4327f2dbd959f654857f58912129b09763329459d08e25547d895ae90fa0f" signed_tx = add_signature_to_transaction(tx, sig) self.assertEqual(signed_tx, expected_signed_tx) tx_obj = decode_transaction(tx) add_signature_to_transaction(tx_obj, sig) self.assertEqual(encode_transaction(tx_obj), expected_signed_tx)
async def create_transaction_skeleton(self, *, to_address, from_address, value=0, nonce=None, gas=None, gas_price=None, data=None): if not validate_address(from_address): raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_from_address', 'message': 'Invalid From Address' }) if to_address is not None and not validate_address(to_address): raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_to_address', 'message': 'Invalid To Address' }) if value: value = parse_int(value) if value is None or value < 0: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_value', 'message': 'Invalid Value' }) # check optional arguments if nonce is None: # check cache for nonce nonce = await self.get_transaction_count(from_address) else: nonce = parse_int(nonce) if nonce is None: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_nonce', 'message': 'Invalid Nonce' }) if data is not None: if isinstance(data, int): data = hex(data) if isinstance(data, str): try: data = data_decoder(data) except binascii.Error: pass if not isinstance(data, bytes): raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_data', 'message': 'Invalid Data field' }) else: data = b'' if gas is None: try: gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=value) except JsonRPCError: # this can occur if sending a transaction to a contract that doesn't match a valid method # and the contract has no default method implemented raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_data', 'message': 'Invalid Data field' }) else: gas = parse_int(gas) if gas is None: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_gas', 'message': 'Invalid Gas' }) if gas_price is None: gas_price = self.application.config['ethereum'].getint( 'default_gasprice', DEFAULT_GASPRICE) else: gas_price = parse_int(gas_price) if gas_price is None: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_gas_price', 'message': 'Invalid Gas Price' }) try: tx = create_transaction(nonce=nonce, gasprice=gas_price, startgas=gas, to=to_address, value=value, data=data, network_id=self.network_id) except InvalidTransaction as e: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_transaction', 'message': str(e) }) if tx.intrinsic_gas_used > gas: raise JsonRPCInvalidParamsError( data={ 'id': 'invalid_transaction', 'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.' .format(tx.intrinsic_gas_used, gas) }) transaction = encode_transaction(tx) return transaction
async def create_transaction_skeleton(self, *, to_address, from_address, value=0, nonce=None, gas=None, gas_price=None, data=None): if not validate_address(from_address): raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address'}) if to_address is not None and not validate_address(to_address): raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address'}) if from_address != from_address.lower() and not checksum_validate_address(from_address): raise JsonRPCInvalidParamsError(data={'id': 'invalid_from_address', 'message': 'Invalid From Address Checksum'}) if to_address is not None and to_address != to_address.lower() and not checksum_validate_address(to_address): raise JsonRPCInvalidParamsError(data={'id': 'invalid_to_address', 'message': 'Invalid To Address Checksum'}) if value: value = parse_int(value) if value is None or value < 0: raise JsonRPCInvalidParamsError(data={'id': 'invalid_value', 'message': 'Invalid Value'}) # check optional arguments # check if we should ignore the given gasprice # NOTE: only meant to be here while cryptokitty fever is pushing # up gas prices... this shouldn't be perminant # anytime the nonce is also set, use the provided gas (this is to # support easier overwriting of transactions) if gas_price is not None and nonce is None: async with self.db: whitelisted = await self.db.fetchrow("SELECT 1 FROM from_address_gas_price_whitelist WHERE address = $1", from_address) if not whitelisted: whitelisted = await self.db.fetchrow("SELECT 1 FROM to_address_gas_price_whitelist WHERE address = $1", to_address) if not whitelisted: gas_price = None if nonce is None: # check cache for nonce nonce = await self.get_transaction_count(from_address) else: nonce = parse_int(nonce) if nonce is None: raise JsonRPCInvalidParamsError(data={'id': 'invalid_nonce', 'message': 'Invalid Nonce'}) if data is not None: if isinstance(data, int): data = hex(data) if isinstance(data, str): try: data = data_decoder(data) except binascii.Error: pass if not isinstance(data, bytes): raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Invalid Data field'}) else: data = b'' if gas is None: try: gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=value) except JsonRPCError: # this can occur if sending a transaction to a contract that doesn't match a valid method # and the contract has no default method implemented raise JsonRPCInvalidParamsError(data={'id': 'invalid_data', 'message': 'Unable to estimate gas for contract call'}) else: gas = parse_int(gas) if gas is None: raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas', 'message': 'Invalid Gas'}) if gas_price is None: # try and use cached gas station gas price gas_station_gas_price = self.redis.get('gas_station_standard_gas_price') if gas_station_gas_price: gas_price = parse_int(gas_station_gas_price) if gas_price is None: gas_price = self.application.config['ethereum'].getint('default_gasprice', DEFAULT_GASPRICE) else: gas_price = parse_int(gas_price) if gas_price is None: raise JsonRPCInvalidParamsError(data={'id': 'invalid_gas_price', 'message': 'Invalid Gas Price'}) try: tx = create_transaction(nonce=nonce, gasprice=gas_price, startgas=gas, to=to_address, value=value, data=data, network_id=self.network_id) except InvalidTransaction as e: raise JsonRPCInvalidParamsError(data={'id': 'invalid_transaction', 'message': str(e)}) if tx.intrinsic_gas_used > gas: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_transaction', 'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.'.format( tx.intrinsic_gas_used, gas)}) transaction = encode_transaction(tx) return {"tx": transaction, "gas": hex(gas), "gas_price": hex(gas_price), "nonce": hex(nonce), "value": hex(value)}
async def create_transaction_skeleton(self, *, to_address, from_address, value=0, nonce=None, gas=None, gas_price=None, data=None, token_address=None): if not validate_address(from_address): raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_from_address', 'message': 'Invalid From Address' }) if to_address is not None and not validate_address(to_address): raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_to_address', 'message': 'Invalid To Address' }) if from_address != from_address.lower( ) and not checksum_validate_address(from_address): raise JsonRPCInvalidParamsError( data={ 'id': 'invalid_from_address', 'message': 'Invalid From Address Checksum' }) if to_address is not None and to_address != to_address.lower( ) and not checksum_validate_address(to_address): raise JsonRPCInvalidParamsError( data={ 'id': 'invalid_to_address', 'message': 'Invalid To Address Checksum' }) # check if we should ignore the given gasprice # NOTE: only meant to be here while cryptokitty fever is pushing # up gas prices... this shouldn't be perminant # anytime the nonce is also set, use the provided gas (this is to # support easier overwriting of transactions) if gas_price is not None and nonce is None: async with self.db: whitelisted = await self.db.fetchrow( "SELECT 1 FROM from_address_gas_price_whitelist WHERE address = $1", from_address) if not whitelisted: whitelisted = await self.db.fetchrow( "SELECT 1 FROM to_address_gas_price_whitelist WHERE address = $1", to_address) if not whitelisted: gas_price = None if gas_price is None: # try and use cached gas station gas price gas_station_gas_price = await self.redis.get( 'gas_station_standard_gas_price') if gas_station_gas_price: gas_price = parse_int(gas_station_gas_price) if gas_price is None: gas_price = config['ethereum'].getint('default_gasprice', DEFAULT_GASPRICE) else: gas_price = parse_int(gas_price) if gas_price is None: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_gas_price', 'message': 'Invalid Gas Price' }) if gas is not None: gas = parse_int(gas) if gas is None: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_gas', 'message': 'Invalid Gas' }) if nonce is None: # check cache for nonce nonce = await self.get_transaction_count(from_address) else: nonce = parse_int(nonce) if nonce is None: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_nonce', 'message': 'Invalid Nonce' }) if data is not None: if isinstance(data, int): data = hex(data) if isinstance(data, str): try: data = data_decoder(data) except binascii.Error: pass if not isinstance(data, bytes): raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_data', 'message': 'Invalid Data field' }) else: data = b'' # flag to force arguments into an erc20 token transfer if token_address is not None: if not validate_address(token_address): raise JsonRPCInvalidParamsError( data={ 'id': 'invalid_token_address', 'message': 'Invalid Token Address' }) if data != b'': raise JsonRPCInvalidParamsError( data={ 'id': 'bad_arguments', 'message': 'Cannot include both data and token_address' }) if isinstance(value, str) and value.lower() == "max": # get the balance in the database async with self.db: value = await self.db.fetchval( "SELECT value FROM token_balances " "WHERE contract_address = $1 AND eth_address = $2", token_address, from_address) if value is None: # get the value from the ethereum node data = "0x70a08231000000000000000000000000" + from_address[ 2:].lower() value = await self.eth.eth_call(to_address=token_address, data=data) value = parse_int(value) if value is None or value < 0: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_value', 'message': 'Invalid Value' }) data = data_decoder( "0xa9059cbb000000000000000000000000{}{:064x}".format( to_address[2:].lower(), value)) token_value = value value = 0 to_address = token_address elif value: if value == "max": network_balance, balance, _, _ = await self.get_balances( from_address) if gas is None: code = await self.eth.eth_getCode(to_address) if code: # we might have to do some work try: gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=0) except JsonRPCError: # no fallback function implemented in the contract means no ether can be sent to it raise JsonRPCInvalidParamsError( data={ 'id': 'invalid_to_address', 'message': 'Cannot send payments to that address' }) attempts = 0 # because the default function could do different things based on the eth sent, we make sure # the value is suitable. if we get different values 3 times abort while True: if attempts > 2: log.warning( "Hit max attempts trying to get max value to send to contract '{}'" .format(to_address)) raise JsonRPCInvalidParamsError( data={ 'id': 'invalid_to_address', 'message': 'Cannot send payments to that address' }) value = balance - (gas_price * gas) try: gas_with_value = await self.eth.eth_estimateGas( from_address, to_address, data=data, value=value) except JsonRPCError: # no fallback function implemented in the contract means no ether can be sent to it raise JsonRPCInvalidParamsError( data={ 'id': 'invalid_to_address', 'message': 'Cannot send payments to that address' }) if gas_with_value != gas: gas = gas_with_value attempts += 1 continue else: break else: # normal address, 21000 gas per transaction gas = 21000 value = balance - (gas_price * gas) else: # preset gas, run with it! value = balance - (gas_price * gas) else: value = parse_int(value) if value is None or value < 0: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_value', 'message': 'Invalid Value' }) if gas is None: try: gas = await self.eth.eth_estimateGas(from_address, to_address, data=data, value=value) except JsonRPCError: # this can occur if sending a transaction to a contract that doesn't match a valid method # and the contract has no default method implemented. # this can also happen if the current state of the blockchain means that submitting the # transaction would fail (abort). if token_address is not None: # when dealing with erc20, this usually means the user's balance for that token isn't # high enough, check that and throw an error if it's the case, and if not fall # back to the standard invalid_data error async with self.db: bal = await self.db.fetchval( "SELECT value FROM token_balances " "WHERE contract_address = $1 AND eth_address = $2", token_address, from_address) if bal is not None: bal = parse_int(bal) if bal < token_value: raise JsonRPCInsufficientFundsError( data={ 'id': 'insufficient_funds', 'message': 'Insufficient Funds' }) raise JsonRPCInvalidParamsError( data={ 'id': 'invalid_data', 'message': 'Unable to estimate gas for contract call' }) # if data is present, buffer gas estimate by 20% if len(data) > 0: gas = int(gas * 1.2) try: tx = create_transaction(nonce=nonce, gasprice=gas_price, startgas=gas, to=to_address, value=value, data=data, network_id=self.network_id) except InvalidTransaction as e: raise JsonRPCInvalidParamsError(data={ 'id': 'invalid_transaction', 'message': str(e) }) if tx.intrinsic_gas_used > gas: raise JsonRPCInvalidParamsError( data={ 'id': 'invalid_transaction', 'message': 'Transaction gas is too low. There is not enough gas to cover minimal cost of the transaction (minimal: {}, got: {}). Try increasing supplied gas.' .format(tx.intrinsic_gas_used, gas) }) transaction = encode_transaction(tx) return { "tx": transaction, "gas": hex(gas), "gas_price": hex(gas_price), "nonce": hex(nonce), "value": hex(token_value) if token_address else hex(value) }
async def _process_transaction_queue(self, ethereum_address): log.debug("processing tx queue for {}".format(ethereum_address)) # check for un-scheduled transactions async with self.db: # get the last block number to use in ethereum calls # to avoid race conditions in transactions being confirmed # on the network before the block monitor sees and updates them in the database last_blocknumber = ( await self.db.fetchval("SELECT blocknumber FROM last_blocknumber")) transactions_out = await self.db.fetch( "SELECT * FROM transactions " "WHERE from_address = $1 " "AND (status = 'new' OR status = 'queued') " "AND r IS NOT NULL " # order by nonce reversed so that .pop() can # be used in the loop below "ORDER BY nonce DESC", ethereum_address) # any time the state of a transaction is changed we need to make # sure those changes cascade down to the receiving address as well # this keeps a list of all the receiving addresses that need to be # checked after the current address's queue has been processed addresses_to_check = set() if transactions_out: # TODO: make sure the block number isn't too far apart from the current # if this is the case then we should just come back later! # get the current network balance for this address balance = await self.eth.eth_getBalance(ethereum_address, block=last_blocknumber or "latest") # get the unconfirmed_txs async with self.db: unconfirmed_txs = await self.db.fetch( "SELECT nonce, value, gas, gas_price FROM transactions " "WHERE from_address = $1 " "AND (status = 'unconfirmed' " "OR (status = 'confirmed' AND blocknumber > $2)) " "ORDER BY nonce", ethereum_address, last_blocknumber or 0) network_nonce = await self.eth.eth_getTransactionCount( ethereum_address, block=last_blocknumber or "latest") if unconfirmed_txs: nonce = unconfirmed_txs[-1]['nonce'] + 1 balance -= sum( parse_int(tx['value']) + (parse_int(tx['gas']) * parse_int(tx['gas_price'])) for tx in unconfirmed_txs) else: # use the nonce from the network nonce = network_nonce # marker for whether a previous transaction had an error (signaling # that all the following should also be an error previous_error = False # for each one, check if we can schedule them yet while transactions_out: transaction = transactions_out.pop() # if there was a previous error in the queue, abort! if previous_error: log.info("Setting tx '{}' to error due to previous error". format(transaction['hash'])) await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add(transaction['to_address']) continue # make sure the nonce is still valid if nonce != transaction[ 'nonce'] and network_nonce != transaction['nonce']: # check if this is an overwrite if transaction['status'] == 'new': async with self.db: old_tx = await self.db.fetchrow( "SELECT * FROM transactions where from_address = $1 AND nonce = $2 AND hash != $3", ethereum_address, transaction['nonce'], transaction['hash']) if old_tx: if old_tx['status'] == 'error': # expected state for overwrites pass elif old_tx['status'] == 'unconfirmed' or old_tx[ 'status'] == 'confirmed': previous_error = True log.info(( "Setting tx '{}' to error due to another unconfirmed transaction" "with nonce ({}) already existing in the system" ).format(transaction['hash'], transaction['nonce'])) await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add( transaction['to_address']) continue else: # two transactions with the same nonce on the queue # lets pick the one with the highest gas price and error the other if transaction['nonce'] > old_tx['nonce']: # lets use this one! log.info(( "Setting tx '{}' to error due to another unconfirmed transaction" "with nonce ({}) already existing in the system" ).format(old_tx['hash'], transaction['nonce'])) await self.update_transaction( old_tx['transaction_id'], 'error') addresses_to_check.add( old_tx['to_address']) # make sure the other transaction is pulled out of the queue try: idx = next(i for i, e in enumerate( transactions_out) if e['transaction_id'] == old_tx['transaction_id']) del transactions_out[idx] except: # old_tx not in the transactions_out list pass else: # we'll use the other one log.info(( "Setting tx '{}' to error due to another unconfirmed transaction" "with nonce ({}) already existing in the system" ).format(old_tx['hash'], transaction['nonce'])) await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add( transaction['to_address']) addresses_to_check.add( transaction['from_address']) # this case is actually pretty weird, so emptying the # transactions_out so we restart the queue check # completely transactions_out = [] continue else: # well this is awkward! may as well let things go on in this case because # it means a transaction in the nonce sequence is missing pass elif transaction['status'] == 'queued': # then this and all the following transactions are now invalid previous_error = True log.info( "Setting tx '{}' to error due to the nonce ({}) not matching the network ({})" .format(transaction['hash'], transaction['nonce'], nonce)) await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add(transaction['to_address']) continue else: # this is a really weird state # it's not clear what should be done here log.error( "Found unconfirmed transaction with out of order nonce for address: {}" .format(ethereum_address)) return value = parse_int(transaction['value']) gas = parse_int(transaction['gas']) gas_price = parse_int(transaction['gas_price']) cost = value + (gas * gas_price) # check if the current balance is high enough to send to the network if balance >= cost: # check if gas price is high enough that it makes sense to send the transaction safe_gas_price = parse_int( await self.redis.get('gas_station_safelow_gas_price')) if safe_gas_price and safe_gas_price > gas_price: log.debug( "Not queuing tx '{}' as current gas price would not support it" .format(transaction['hash'])) # retry this address in a minute manager_dispatcher.process_transaction_queue( ethereum_address).delay(60) # abort the rest of the processing after sending PNs for any "new" transactions while transaction: if transaction['status'] == 'new': await self.update_transaction( transaction['transaction_id'], 'queued') transaction = transactions_out.pop( ) if transactions_out else None break # if so, send the transaction # create the transaction data = data_decoder( transaction['data']) if transaction['data'] else b'' tx = create_transaction(nonce=transaction['nonce'], value=value, gasprice=gas_price, startgas=gas, to=transaction['to_address'], data=data, v=parse_int(transaction['v']), r=parse_int(transaction['r']), s=parse_int(transaction['s'])) # make sure the signature was valid if data_encoder(tx.sender) != ethereum_address: # signature is invalid for the user log.error( "ERROR signature invalid for sender of tx: {}". format(transaction['hash'])) log.error("queue: {}, db: {}, tx: {}".format( ethereum_address, transaction['from_address'], data_encoder(tx.sender))) previous_error = True addresses_to_check.add(transaction['to_address']) await self.update_transaction( transaction['transaction_id'], 'error') continue # send the transaction try: tx_encoded = encode_transaction(tx) await self.eth.eth_sendRawTransaction(tx_encoded) await self.update_transaction( transaction['transaction_id'], 'unconfirmed') except JsonRPCError as e: # if something goes wrong with sending the transaction # simply abort for now. # TODO: depending on error, just break and queue to retry later log.error( "ERROR sending queued transaction: {}".format( e.format())) if e.message and (e.message.startswith( "Transaction nonce is too low" ) or e.message.startswith( "Transaction with the same hash was already imported" )): existing_tx = await self.eth.eth_getTransactionByHash( transaction['hash']) if existing_tx: if existing_tx['blockNumber']: await self.update_transaction( transaction['transaction_id'], 'confirmed') else: await self.update_transaction( transaction['transaction_id'], 'unconfirmed') continue previous_error = True await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add(transaction['to_address']) continue # adjust the balance values for checking the other transactions balance -= cost if nonce == transaction['nonce']: nonce += 1 continue else: # make sure the pending_balance would support this transaction # otherwise there's no way this transaction will be able to # be send, so trigger a failure on all the remaining transactions async with self.db: transactions_in = await self.db.fetch( "SELECT * FROM transactions " "WHERE to_address = $1 " "AND (" "(status = 'new' OR status = 'queued' OR status = 'unconfirmed') " "OR (status = 'confirmed' AND blocknumber > $2))", ethereum_address, last_blocknumber or 0) # TODO: test if loops in the queue chain are problematic pending_received = sum( (parse_int(p['value']) or 0) for p in transactions_in) if balance + pending_received < cost: previous_error = True log.info( "Setting tx '{}' to error due to insufficient pending balance" .format(transaction['hash'])) await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add(transaction['to_address']) continue else: if any(t['blocknumber'] is not None and t['blocknumber'] > last_blocknumber for t in transactions_in): addresses_to_check.add(ethereum_address) # there's no reason to continue on here since all the # following transaction in the queue cannot be processed # until this one is # but we still need to send PNs for any "new" transactions while transaction: if transaction['status'] == 'new': await self.update_transaction( transaction['transaction_id'], 'queued') transaction = transactions_out.pop( ) if transactions_out else None break for address in addresses_to_check: # make sure we don't try process any contract deployments if address != "0x": manager_dispatcher.process_transaction_queue(address) if transactions_out: manager_dispatcher.process_transaction_queue(ethereum_address)
async def sanity_check(self, frequency): async with self.db: rows = await self.db.fetch( "SELECT DISTINCT from_address FROM transactions WHERE (status = 'unconfirmed' OR status = 'queued' OR status = 'new') " "AND v IS NOT NULL AND created < (now() AT TIME ZONE 'utc') - interval '3 minutes'" ) rows2 = await self.db.fetch( "WITH t1 AS (SELECT DISTINCT from_address FROM transactions WHERE (status = 'new' OR status = 'queued') AND v IS NOT NULL), " "t2 AS (SELECT from_address, COUNT(*) FROM transactions WHERE (status = 'unconfirmed' AND v IS NOT NULL) GROUP BY from_address) " "SELECT t1.from_address FROM t1 LEFT JOIN t2 ON t1.from_address = t2.from_address WHERE t2.count IS NULL;" ) if rows or rows2: log.debug( "sanity check found {} addresses with potential problematic transactions" .format(len(rows) + len(rows2))) rows = set([row['from_address'] for row in rows ]).union(set([row['from_address'] for row in rows2])) addresses_to_check = set() old_and_unconfirmed = [] for ethereum_address in rows: # check on queued transactions async with self.db: queued_transactions = await self.db.fetch( "SELECT * FROM transactions " "WHERE from_address = $1 " "AND (status = 'new' OR status = 'queued') AND v IS NOT NULL", ethereum_address) if queued_transactions: # make sure there are pending incoming transactions async with self.db: incoming_transactions = await self.db.fetch( "SELECT * FROM transactions " "WHERE to_address = $1 " "AND (status = 'unconfirmed' OR status = 'queued' OR status = 'new')", ethereum_address) if not incoming_transactions: log.error( "ERROR: {} has transactions in it's queue, but no unconfirmed transactions!" .format(ethereum_address)) # trigger queue processing as last resort addresses_to_check.add(ethereum_address) else: # check health of the incoming transaction for transaction in incoming_transactions: if transaction['v'] is None: try: tx = await self.eth.eth_getTransactionByHash( transaction['hash']) except: log.exception( "Error getting transaction {} in sanity check", transaction['hash']) continue if tx is None: log.warning( "external transaction (id: {}) no longer found on nodes" .format(transaction['transaction_id'])) await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add(ethereum_address) elif tx['blockNumber'] is not None: log.warning( "external transaction (id: {}) confirmed on node, but wasn't confirmed in db" .format(transaction['transaction_id'])) await self.update_transaction( transaction['transaction_id'], 'confirmed') addresses_to_check.add(ethereum_address) # no need to continue with dealing with unconfirmed transactions if there are queued ones continue async with self.db: unconfirmed_transactions = await self.db.fetch( "SELECT * FROM transactions " "WHERE from_address = $1 " "AND status = 'unconfirmed' AND v IS NOT NULL", ethereum_address) if unconfirmed_transactions: for transaction in unconfirmed_transactions: # check on unconfirmed transactions first if transaction['status'] == 'unconfirmed': # we neehed to check the true status of unconfirmed transactions # as the block monitor may be inbetween calls and not have seen # this transaction to mark it as confirmed. try: tx = await self.eth.eth_getTransactionByHash( transaction['hash']) except: log.exception( "Error getting transaction {} in sanity check", transaction['hash']) continue # sanity check to make sure the tx still exists if tx is None: # if not, try resubmit # NOTE: it may just be an issue with load balanced nodes not seeing all pending transactions # so we don't want to adjust the status of the transaction at all at this stage value = parse_int(transaction['value']) gas = parse_int(transaction['gas']) gas_price = parse_int(transaction['gas_price']) data = data_decoder( transaction['data'] ) if transaction['data'] else b'' tx = create_transaction( nonce=transaction['nonce'], value=value, gasprice=gas_price, startgas=gas, to=transaction['to_address'], data=data, v=parse_int(transaction['v']), r=parse_int(transaction['r']), s=parse_int(transaction['s'])) if calculate_transaction_hash( tx) != transaction['hash']: log.warning( "error resubmitting transaction {}: regenerating tx resulted in a different hash" .format(transaction['hash'])) else: tx_encoded = encode_transaction(tx) try: await self.eth.eth_sendRawTransaction( tx_encoded) addresses_to_check.add( transaction['from_address']) except Exception as e: # note: usually not critical, don't panic log.warning( "error resubmitting transaction {}: {}" .format(transaction['hash'], str(e))) elif tx['blockNumber'] is not None: # confirmed! update the status await self.update_transaction( transaction['transaction_id'], 'confirmed') addresses_to_check.add(transaction['from_address']) addresses_to_check.add(transaction['to_address']) else: old_and_unconfirmed.append(transaction['hash']) if len(old_and_unconfirmed): log.warning( "WARNING: {} transactions are old and unconfirmed!".format( len(old_and_unconfirmed))) for address in addresses_to_check: # make sure we don't try process any contract deployments if address != "0x": manager_dispatcher.process_transaction_queue(address) if frequency: manager_dispatcher.sanity_check(frequency).delay(frequency)
async def _process_transaction_queue(self, ethereum_address): log.info("processing tx queue for {}".format(ethereum_address)) # check for un-scheduled transactions async with self.db: # get the last block number to use in ethereum calls # to avoid race conditions in transactions being confirmed # on the network before the block monitor sees and updates them in the database last_blocknumber = ( await self.db.fetchval("SELECT blocknumber FROM last_blocknumber")) transactions_out = await self.db.fetch( "SELECT * FROM transactions " "WHERE from_address = $1 " "AND (status is NULL OR status = 'queued') " "AND r IS NOT NULL " # order by nonce reversed so that .pop() can # be used in the loop below "ORDER BY nonce DESC", ethereum_address) # any time the state of a transaction is changed we need to make # sure those changes cascade down to the receiving address as well # this keeps a list of all the receiving addresses that need to be # checked after the current address's queue has been processed addresses_to_check = set() if transactions_out: # TODO: make sure the block number isn't too far apart from the current # if this is the case then we should just come back later! # get the current network balance for this address balance = await self.eth.eth_getBalance(ethereum_address, block=last_blocknumber or "latest") # get the unconfirmed_txs async with self.db: unconfirmed_txs = await self.db.fetch( "SELECT nonce, value, gas, gas_price FROM transactions " "WHERE from_address = $1 " "AND (status = 'unconfirmed' " "OR (status = 'confirmed' AND blocknumber > $2)) " "ORDER BY nonce", ethereum_address, last_blocknumber or 0) if unconfirmed_txs: nonce = unconfirmed_txs[-1]['nonce'] + 1 balance -= sum( parse_int(tx['value']) + (parse_int(tx['gas']) * parse_int(tx['gas_price'])) for tx in unconfirmed_txs) else: # use the nonce from the network nonce = await self.eth.eth_getTransactionCount( ethereum_address, block=last_blocknumber or "latest") # marker for whether a previous transaction had an error (signaling # that all the following should also be an error previous_error = False # for each one, check if we can schedule them yet while transactions_out: transaction = transactions_out.pop() # if there was a previous error in the queue, abort! if previous_error: log.info("Setting tx '{}' to error due to previous error". format(transaction['hash'])) await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add(transaction['to_address']) continue # make sure the nonce is still valid if nonce != transaction['nonce']: # then this and all the following transactions are now invalid previous_error = True log.info( "Setting tx '{}' to error due to the nonce ({}) not matching the network ({})" .format(transaction['hash'], transaction['nonce'], nonce)) await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add(transaction['to_address']) continue value = parse_int(transaction['value']) gas = parse_int(transaction['gas']) gas_price = parse_int(transaction['gas_price']) cost = value + (gas * gas_price) # check if the current balance is high enough to send to the network if balance >= cost: # if so, send the transaction # create the transaction data = data_decoder( transaction['data']) if transaction['data'] else b'' tx = create_transaction(nonce=nonce, value=value, gasprice=gas_price, startgas=gas, to=transaction['to_address'], data=data, v=parse_int(transaction['v']), r=parse_int(transaction['r']), s=parse_int(transaction['s'])) # make sure the signature was valid if data_encoder(tx.sender) != ethereum_address: # signature is invalid for the user log.error( "ERROR signature invalid for sender of tx: {}". format(transaction['hash'])) log.error("queue: {}, db: {}, tx: {}".format( ethereum_address, transaction['from_address'], data_encoder(tx.sender))) previous_error = True addresses_to_check.add(transaction['to_address']) await self.update_transaction( transaction['transaction_id'], 'error') continue # send the transaction try: tx_encoded = encode_transaction(tx) await self.eth.eth_sendRawTransaction(tx_encoded) await self.update_transaction( transaction['transaction_id'], 'unconfirmed') except JsonRPCError as e: # if something goes wrong with sending the transaction # simply abort for now. # TODO: depending on error, just break and queue to retry later log.error( "ERROR sending queued transaction: {}".format( e.format())) previous_error = True await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add(transaction['to_address']) continue # adjust the balance values for checking the other transactions balance -= cost nonce += 1 continue else: # make sure the pending_balance would support this transaction # otherwise there's no way this transaction will be able to # be send, so trigger a failure on all the remaining transactions async with self.db: transactions_in = await self.db.fetch( "SELECT * FROM transactions " "WHERE to_address = $1 " "AND (" "(status is NULL OR status = 'queued' OR status = 'unconfirmed') " "OR (status = 'confirmed' AND blocknumber > $2))", ethereum_address, last_blocknumber or 0) # TODO: test if loops in the queue chain are problematic pending_received = sum( (parse_int(p['value']) or 0) for p in transactions_in) if balance + pending_received < cost: previous_error = True log.info( "Setting tx '{}' to error due to insufficient pending balance" .format(transaction['hash'])) await self.update_transaction( transaction['transaction_id'], 'error') addresses_to_check.add(transaction['to_address']) continue else: if any(t['blocknumber'] is not None and t['blocknumber'] > last_blocknumber for t in transactions_in): addresses_to_check.add(ethereum_address) # there's no reason to continue on here since all the # following transaction in the queue cannot be processed # until this one is # but we still need to send PNs for any "new" transactions while transaction: if transaction['status'] is None: await self.update_transaction( transaction['transaction_id'], 'queued') transaction = transactions_out.pop( ) if transactions_out else None break for address in addresses_to_check: # make sure we don't try process any contract deployments if address != "0x": self.tasks.process_transaction_queue(address) if transactions_out: self.tasks.process_transaction_queue(ethereum_address)
def do_POST(self): # TODO: figure out why read is blocking here data = self.rfile.read(len(self.rfile.peek())) data = data.decode('utf-8') data = json.loads(data) if self.path == "/v1/tx/skel": gas_price = parse_int( data['gas_price']) if 'gas_price' in data else DEFAULT_GASPRICE gas = parse_int(data['gas']) if 'gas' in data else DEFAULT_STARTGAS nonce = parse_int(data['nonce']) if 'nonce' in data else 0 if 'value' not in data or 'from' not in data or 'to' not in data: self.write_data( 400, { 'errors': [{ 'id': 'bad_arguments', 'message': 'Bad Arguments' }] }) return value = parse_int(data['value']) to_address = data['to'] from_address = data['from'] if not validate_address(to_address): self.write_data( 400, { 'errors': [{ 'id': 'invalid_to_address', 'message': 'Invalid To Address' }] }) return if not validate_address(from_address): self.write_data( 400, { 'errors': [{ 'id': 'invalid_from_address', 'message': 'Invalid From Address' }] }) return tx = create_transaction(nonce=nonce, gasprice=gas_price, startgas=gas, to=to_address, value=value) transaction = encode_transaction(tx) self.write_data( 200, { "tx_data": { "nonce": hex(nonce), "from": from_address, "to": to_address, "value": hex(value), "startGas": hex(gas), "gasPrice": hex(gas_price) }, "tx": transaction }) elif self.path == "/v1/tx": tx = decode_transaction(data['tx']) if 'signature' in data: sig = data_decoder(data['signature']) add_signature_to_transaction(tx, sig) self.write_data(200, {"tx_hash": data_encoder(tx.hash)}) else: self.write_data(404)