def __init__(self): configure_logger(log) if 'monitor' in config: node_url = config['monitor']['url'] else: log.warning("monitor using config['ethereum'] node") node_url = config['ethereum']['url'] self.eth = JsonRPCClient(node_url, connect_timeout=5.0, request_timeout=10.0) # filter health processes depend on some of the calls failing on the first time # so we have a separate client to handle those self.filter_eth = JsonRPCClient(node_url, force_instance=True, connect_timeout=10.0, request_timeout=60.0) self._check_schedule = None self._poll_schedule = None self._sanity_check_schedule = None self._block_checking_process = None self._filter_poll_process = None self._sanity_check_process = None self._process_unconfirmed_transactions_process = None self._new_pending_transaction_filter_id = None self._last_saw_new_block = asyncio.get_event_loop().time() self._shutdown = False self._lastlog = 0 self._blocktimes = []
def __init__(self): configure_logger(log) if 'monitor' in config: node_url = config['monitor']['url'] else: log.warning("monitor using config['ethereum'] node") node_url = config['ethereum']['url'] self.eth = JsonRPCClient(node_url, should_retry=True)
def process_config(self): config = super().process_config() if 'ETHEREUM_NODE_URL' in os.environ: config['ethereum'] = {'url': os.environ['ETHEREUM_NODE_URL']} if 'DEFAULT_GASPRICE' in os.environ: if 'ethereum' not in config: config['ethereum'] = {} config['ethereum']['default_gasprice'] = os.environ[ 'DEFAULT_GASPRICE'] if 'ethereum' in config: if 'ETHEREUM_NETWORK_ID' in os.environ: config['ethereum']['network_id'] = os.environ[ 'ETHEREUM_NETWORK_ID'] else: config['ethereum'][ 'network_id'] = self.asyncio_loop.run_until_complete( to_asyncio_future( JsonRPCClient( config['ethereum']['url']).net_version())) configure_logger(services_log) return config
def __init__(self, *args, listener_id="block_monitor", **kwargs): # so DatabaseMixin works self.application = self super().__init__([], *args, listener_id=listener_id, **kwargs) configure_logger(log) if 'monitor' in self.config: node_url = self.config['monitor']['url'] else: log.warning("monitor using config['ethereum'] node") node_url = self.config['ethereum']['url'] self.eth = JsonRPCClient(node_url) self._check_schedule = None self._poll_schedule = None self._block_checking_process = None self._filter_poll_process = None self._lastlog = 0 self.tasks = TaskDispatcher(self.task_listener)
async def test_getLogs_unknown_block_number_handling(self, *, parity): """Ensures that the block number check is working correctly""" client = JsonRPCClient(parity.dsn()['url']) block_number = await client.eth_blockNumber() logs = await client.eth_getLogs(fromBlock=block_number + 2) self.assertEqual(logs, []) self.assertGreaterEqual( await client.eth_blockNumber(), block_number + 2, "eth_getLogs did not wait until the node's block number caught up to the requested block" ) logs = await client.eth_getLogs(fromBlock=block_number, toBlock=block_number + 6) self.assertEqual(logs, []) self.assertGreaterEqual( await client.eth_blockNumber(), block_number + 6, "eth_getLogs did not wait until the node's block number caught up to the requested block" ) logs = await client.eth_getLogs(toBlock=block_number + 10) self.assertEqual(logs, []) self.assertGreaterEqual( await client.eth_blockNumber(), block_number + 10, "eth_getLogs did not wait until the node's block number caught up to the requested block" ) logs = await client.eth_getLogs(toBlock=block_number + 20, validate_block_number=False) self.assertLess( await client.eth_blockNumber(), block_number + 20, "eth_getLogs unexpectidly waited until the block number caught up to the requested block" ) self.assertEqual( logs, [], "parity started returning something other than [] for eth_getLogs when block params are too high" ) await client.close()
def __init__(self, *args, **kwargs): super().__init__([(CollectiblesProcessingHandler, )], *args, listener_id=self.__class__.__name__, **kwargs) self.eth = JsonRPCClient(self.config['ethereum']['url'], should_retry=False) self.ioloop.add_callback(self.process_block)
async def test_aiohttp_jsonrpc_client(self, *, parity): client = JsonRPCClient(parity.dsn()['url'], client_cls=AIOHTTPClient) block_number = await client.eth_blockNumber() balance = await client.eth_getBalance(FAUCET_ADDRESS) self.assertEqual( balance, 1606938044258990275541962092341162602522202993782792835301376) await client.close()
async def test_unknown_block_number_handling(self, *, parity): client = JsonRPCClient(parity.dsn()['url']) block_number = await client.eth_blockNumber() balance = await client.eth_getBalance(FAUCET_ADDRESS, block=block_number + 2) self.assertEqual( balance, 1606938044258990275541962092341162602522202993782792835301376) await client.close()
async def faucet(self, to, value, *, from_private_key=FAUCET_PRIVATE_KEY, startgas=None, gasprice=DEFAULT_GASPRICE, nonce=None, data=b"", wait_on_confirmation=True): if isinstance(from_private_key, str): from_private_key = data_decoder(from_private_key) from_address = private_key_to_address(from_private_key) ethclient = JsonRPCClient(config['ethereum']['url']) to = data_decoder(to) if len(to) not in (20, 0): raise Exception( 'Addresses must be 20 or 0 bytes long (len was {})'.format( len(to))) if nonce is None: nonce = await ethclient.eth_getTransactionCount(from_address) balance = await ethclient.eth_getBalance(from_address) if startgas is None: startgas = await ethclient.eth_estimateGas(from_address, to, data=data, nonce=nonce, value=value, gasprice=gasprice) tx = Transaction(nonce, gasprice, startgas, to, value, data, 0, 0, 0) if balance < (tx.value + (tx.startgas * tx.gasprice)): raise Exception("Faucet doesn't have enough funds") tx.sign(from_private_key) tx_encoded = data_encoder(rlp.encode(tx, Transaction)) tx_hash = await ethclient.eth_sendRawTransaction(tx_encoded) while wait_on_confirmation: resp = await ethclient.eth_getTransactionByHash(tx_hash) if resp is None or resp['blockNumber'] is None: await asyncio.sleep(0.1) else: break if to == b'': print("contract address: {}".format(data_encoder(tx.creates))) return tx_hash
async def deploy_contract(self, bytecode, *, from_private_key=FAUCET_PRIVATE_KEY, startgas=None, gasprice=DEFAULT_GASPRICE, wait_on_confirmation=True): if isinstance(from_private_key, str): from_private_key = data_decoder(from_private_key) from_address = private_key_to_address(from_private_key) ethclient = JsonRPCClient(config['ethereum']['url']) nonce = await ethclient.eth_getTransactionCount(from_address) balance = await ethclient.eth_getBalance(from_address) gasestimate = await ethclient.eth_estimateGas(from_address, '', data=bytecode, nonce=nonce, value=0, gasprice=gasprice) if startgas is None: startgas = gasestimate elif gasestimate > startgas: raise Exception( "Estimated gas usage is larger than the provided gas") tx = Transaction(nonce, gasprice, startgas, '', 0, bytecode, 0, 0, 0) if balance < (tx.value + (tx.startgas * tx.gasprice)): raise Exception("Faucet doesn't have enough funds") tx.sign(from_private_key) tx_encoded = data_encoder(rlp.encode(tx, Transaction)) tx_hash = await ethclient.eth_sendRawTransaction(tx_encoded) contract_address = data_encoder(tx.creates) while wait_on_confirmation: resp = await ethclient.eth_getTransactionByHash(tx_hash) if resp is None or resp['blockNumber'] is None: await asyncio.sleep(0.1) else: code = await ethclient.eth_getCode(contract_address) if code == '0x': raise Exception("Failed to deploy contract") break return tx_hash, contract_address
async def test_transaction_overwrite_spam(self, *, ethminer, parity, monitor, push_client): no_to_spam = 10 # make sure no blocks are confirmed ethminer.pause() # set up pn registrations async with self.pool.acquire() as con: await con.fetch( "INSERT INTO notification_registrations (service, registration_id, toshi_id, eth_address) VALUES ($1, $2, $3, $4)", 'gcm', TEST_GCM_ID, TEST_ID_ADDRESS, TEST_WALLET_ADDRESS) # send initial tx tx1 = await self.get_tx_skel(FAUCET_PRIVATE_KEY, TEST_WALLET_ADDRESS, 10**18) txs = [] for i in range(no_to_spam): tx = await self.get_tx_skel(FAUCET_PRIVATE_KEY, TEST_WALLET_ADDRESS, i) txs.append(sign_transaction(tx, FAUCET_PRIVATE_KEY)) tx1_hash = await self.sign_and_send_tx(FAUCET_PRIVATE_KEY, tx1) # wait for tx PN await push_client.get() # spam send txs manually rpcclient = JsonRPCClient(parity.dsn()['url']) for ntx in txs: await rpcclient.eth_sendRawTransaction(ntx) # force the pending transaction filter polling to # run after each new transaction is posted await monitor.filter_poll() # we expect two pns for each overwrite await push_client.get() await push_client.get() async with self.pool.acquire() as con: tx1_row = await con.fetchrow( "SELECT * FROM transactions WHERE hash = $1", tx1_hash) tx_rows = await con.fetchrow("SELECT COUNT(*) FROM transactions") tx_rows_error = await con.fetchrow( "SELECT COUNT(*) FROM transactions WHERE status = 'error'") self.assertEqual(tx1_row['status'], 'error') self.assertEqual(tx_rows['count'], no_to_spam + 1) self.assertEqual(tx_rows_error['count'], no_to_spam)
async def test_bulk(self, *, parity): client = JsonRPCClient(parity.dsn()['url']) bulk = client.bulk() f1 = bulk.eth_blockNumber() f2 = bulk.eth_getBalance(FAUCET_ADDRESS) f3 = bulk.eth_gasPrice() f4 = bulk.eth_getBalance("0x0000000000000000000000000000000000000000") f5 = bulk.eth_getBalance(FAUCET_ADDRESS, block=100000000) results = await bulk.execute() self.assertEqual(f1.result(), results[0]) self.assertEqual(f2.result(), results[1]) self.assertEqual(f3.result(), results[2]) self.assertEqual(f4.result(), results[3]) try: f5.result() self.fail("expected exception") except JsonRPCError as e: self.assertEqual(e.message, "Unknown block number") except Exception as e: self.fail("unexpected exception: {}".format(e)) await client.close()
async def test_deploy_contract(self, *, node): client = JsonRPCClient(node.dsn()['url']) sourcecode = b"contract greeter{string greeting;function greeter(string _greeting) public{greeting=_greeting;}function greet() constant returns (string){return greeting;}}" #source_fn = os.path.join(node.get_data_directory(), 'greeting.sol') #with open(source_fn, 'wb') as wf: # wf.write(sourcecode) source_fn = '<stdin>' contract_name = 'greeter' constructor_args = [b'hello world!'] args = ['solc', '--combined-json', 'bin,abi', '--add-std'] # , source_fn] #output = subprocess.check_output(args, stderr=subprocess.PIPE) process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, stderrdata = process.communicate(input=sourcecode) output = json_decode(output) contract = output['contracts']['{}:{}'.format(source_fn, contract_name)] bytecode = data_decoder(contract['bin']) contract_interface = json_decode(contract['abi']) translator = ContractTranslator(contract_interface) constructor_call = translator.encode_constructor_arguments( constructor_args) bytecode += constructor_call tx_hash, contract_address = await self.deploy_contract(bytecode) tx_receipt = await client.eth_getTransactionReceipt(tx_hash) self.assertIsNotNone(tx_receipt) code = await client.eth_getCode(contract_address) self.assertIsNotNone(code) self.assertNotEqual(data_decoder(code), b'') # call the contract and check the result res = await client.eth_call( from_address='0x39bf9e501e61440b4b268d7b2e9aa2458dd201bb', to_address=contract_address, data=sha3('greet()')) result = translator.decode_function_result('greet', data_decoder(res)) self.assertEqual(result[0], constructor_args[0])
async def test_raw_deploy_contract(self, *, parity): """Tests that sending a raw transaction with a contract deployment works""" # contract data data = "0x6060604052341561000c57fe5b6040516102b83803806102b8833981016040528080518201919050505b806000908051906020019061003f929190610047565b505b506100ec565b828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f1061008857805160ff19168380011785556100b6565b828001600101855582156100b6579182015b828111156100b557825182559160200191906001019061009a565b5b5090506100c391906100c7565b5090565b6100e991905b808211156100e55760008160009055506001016100cd565b5090565b90565b6101bd806100fb6000396000f30060606040526000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063cfae32171461003b575bfe5b341561004357fe5b61004b6100d4565b604051808060200182810382528381815181526020019150805190602001908083836000831461009a575b80518252602083111561009a57602082019150602081019050602083039250610076565b505050905090810190601f1680156100c65780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b6100dc61017d565b60008054600181600116156101000203166002900480601f0160208091040260200160405190810160405280929190818152602001828054600181600116156101000203166002900480156101725780601f1061014757610100808354040283529160200191610172565b820191906000526020600020905b81548152906001019060200180831161015557829003601f168201915b505050505090505b90565b6020604051908101604052806000815250905600a165627a7a72305820493059270656b40625319934bd6e91b0e68cf32c54c099dfc6cf540e40c91b9500290000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000c68656c6c6f20776f726c64210000000000000000000000000000000000000000" resp = await self.fetch("/tx/skel", method="POST", body={ "from": FAUCET_ADDRESS, "data": data }) self.assertEqual(resp.code, 200) body = json_decode(resp.body) tx = sign_transaction(body['tx'], FAUCET_PRIVATE_KEY) resp = await self.fetch("/tx", method="POST", body={"tx": tx}) self.assertEqual(resp.code, 200, resp.body) await self.wait_on_tx_confirmation(json_decode(resp.body)['tx_hash']) # test that contract txs from outside are handled correctly resp = await self.fetch_signed( "/apn/register", signing_key=FAUCET_PRIVATE_KEY, method="POST", body={"registration_id": "blahblahblah"}) resp = await self.fetch("/tx/skel", method="POST", body={ "from": FAUCET_ADDRESS, "data": data }) self.assertEqual(resp.code, 200) body = json_decode(resp.body) tx = sign_transaction(body['tx'], FAUCET_PRIVATE_KEY) # deploy manually rpcclient = JsonRPCClient(parity.dsn()['url']) tx_hash = await rpcclient.eth_sendRawTransaction(tx) await self.wait_on_tx_confirmation(tx_hash) await asyncio.sleep( 5) # make sure the monitor has had a chance to process this async with self.pool.acquire() as con: rows = await con.fetch( "SELECT * FROM transactions WHERE hash = $1", tx_hash) self.assertEqual(len(rows), 1)
async def test_resend_old_after_overwrite(self, *, ethminer, parity, monitor, push_client): # make sure no blocks are confirmed for the meantime ethminer.pause() # set up pn registrations async with self.pool.acquire() as con: await con.fetch( "INSERT INTO notification_registrations (service, registration_id, toshi_id, eth_address) VALUES ($1, $2, $3, $4)", 'gcm', TEST_GCM_ID, TEST_ID_ADDRESS, TEST_WALLET_ADDRESS) # get tx skeleton tx1 = await self.get_tx_skel(FAUCET_PRIVATE_KEY, TEST_WALLET_ADDRESS, 10**18) tx2 = await self.get_tx_skel(FAUCET_PRIVATE_KEY, TEST_WALLET_ADDRESS, 0) self.assertEqual( decode_transaction(tx1).nonce, decode_transaction(tx2).nonce) # sign and send tx1_hash = await self.sign_and_send_tx(FAUCET_PRIVATE_KEY, tx1) # wait for tx PN await push_client.get() # send tx2 manually rpcclient = JsonRPCClient(parity.dsn()['url']) tx2_hash = await rpcclient.eth_sendRawTransaction( sign_transaction(tx2, FAUCET_PRIVATE_KEY)) await monitor.filter_poll() _, pn = await push_client.get() _, pn = await push_client.get() # resend tx1 manually tx1_hash = await rpcclient.eth_sendRawTransaction( sign_transaction(tx1, FAUCET_PRIVATE_KEY)) await monitor.filter_poll() _, pn = await push_client.get() _, pn = await push_client.get() async with self.pool.acquire() as con: tx1_row = await con.fetchrow( "SELECT * FROM transactions WHERE hash = $1", tx1_hash) tx2_row = await con.fetchrow( "SELECT * FROM transactions WHERE hash = $1", tx2_hash) self.assertEqual(tx1_row['status'], 'unconfirmed') self.assertEqual(tx2_row['status'], 'error')
def prepare_ethereum_jsonrpc_client(config): if 'url' in config: url = config['url'] elif 'host' in config: ssl = config.get('ssl', 'false') if ssl is True or (isinstance(ssl, str) and ssl.lower() == 'true'): protocol = 'https://' else: protocol = 'http://' port = config.get('port', '8545') host = config.get('host', 'localhost') path = config.get('path', '/') if not path.startswith('/'): path = "/{}".format(path) url = "{}{}:{}{}".format(protocol, host, port, path) return JsonRPCClient(url)
def extra_service_config(): config.set_from_os_environ('ethereum', 'url', 'ETHEREUM_NODE_URL') config.set_from_os_environ('monitor', 'url', 'MONITOR_ETHEREUM_NODE_URL') if 'ethereum' in config: if 'ETHEREUM_NETWORK_ID' in os.environ: config['ethereum']['network_id'] = os.environ[ 'ETHEREUM_NETWORK_ID'] else: config['ethereum']['network_id'] = asyncio.get_event_loop( ).run_until_complete( JsonRPCClient(config['ethereum']['url']).net_version()) # push service config config.set_from_os_environ('pushserver', 'url', 'PUSH_URL') config.set_from_os_environ('pushserver', 'username', 'PUSH_USERNAME') config.set_from_os_environ('pushserver', 'password', 'PUSH_PASSWORD') config.set_from_os_environ('gcm', 'server_key', 'GCM_SERVER_KEY')
def __init__(self): configure_logger(log) if 'monitor' in config: node_url = config['monitor']['url'] else: log.warning("monitor using config['ethereum'] node") node_url = config['ethereum']['url'] self.eth = JsonRPCClient(node_url, should_retry=False) self._check_schedule = None self._poll_schedule = None self._sanity_check_schedule = None self._block_checking_process = None self._filter_poll_process = None self._sanity_check_process = None self._process_unconfirmed_transactions_process = None self._lastlog = 0
def process_config(self): config = super().process_config() if 'ETHEREUM_NODE_URL' in os.environ: config['ethereum'] = {'url': os.environ['ETHEREUM_NODE_URL']} if 'MONITOR_ETHEREUM_NODE_URL' in os.environ: config['monitor'] = { 'url': os.environ['MONITOR_ETHEREUM_NODE_URL'] } if 'ethereum' in config: if 'ETHEREUM_NETWORK_ID' in os.environ: config['ethereum']['network_id'] = os.environ[ 'ETHEREUM_NETWORK_ID'] else: config['ethereum'][ 'network_id'] = self.asyncio_loop.run_until_complete( to_asyncio_future( JsonRPCClient( config['ethereum']['url']).net_version())) return config
async def test_tx_overwrite(self, *, ethminer, parity, push_client): """Tests that if a transaction with the same nonce and one the system knows about is sent from outside of the system and included in the block, that the error handling picks this up correctly""" # start 2nd parity server p2 = ParityServer(bootnodes=parity.dsn()['node']) e2 = EthMiner(jsonrpc_url=p2.dsn()['url'], debug=False) rpcclient2 = JsonRPCClient(p2.dsn()['url']) addr1, pk1 = TEST_ADDRESSES[0] addr2, pk2 = TEST_ADDRESSES[1] addr3, pk3 = TEST_ADDRESSES[2] val = 1000 * 10**18 # send funds to address1 f_tx_hash = await self.send_tx(FAUCET_PRIVATE_KEY, TEST_ADDRESS_1, val) # make sure sync is done while True: data2 = await rpcclient2.eth_getTransactionByHash(f_tx_hash) if data2 and data2['blockNumber'] is not None: break await asyncio.sleep(1) # make sure no blocks are mined ethminer.pause() e2.pause() # make sure transactions are "interesting" to the monitory async with self.pool.acquire() as con: for addr, pk in TEST_ADDRESSES[:3]: await con.fetch( "INSERT INTO notification_registrations (service, registration_id, toshi_id, eth_address) VALUES ($1, $2, $3, $4)", 'gcm', "abc", addr, addr) # create two transactions with the same nonce and submit them both tx1 = await self.get_tx_skel(pk1, addr2, int(val / 3)) tx2 = await self.get_tx_skel(pk1, addr3, int(val / 3), nonce=decode_transaction(tx1).nonce) tx2 = sign_transaction(tx2, pk1) tx_hash_2 = await rpcclient2.eth_sendRawTransaction(tx2) tx_hash_1 = await self.sign_and_send_tx(pk1, tx1) # start mining again e2.start() # wait for one of the two transactions to complete try: while True: async with self.pool.acquire() as con: tx1_row = await con.fetchrow( "SELECT * FROM transactions WHERE hash = $1", tx_hash_1) tx2_row = await con.fetchrow( "SELECT * FROM transactions WHERE hash = $1", tx_hash_2) if tx2_row is not None and tx2_row['status'] == 'confirmed': # good! break if tx1_row is not None and tx1_row['status'] == 'confirmed': self.assertFail( "tx1 confirmed, expected tx1 overwrite and tx2 confirmed" ) await asyncio.sleep(1) finally: e2.stop() p2.stop()
class HealthMonitor: def __init__(self): configure_logger(log) if 'monitor' in config: node_url = config['monitor']['url'] else: log.warning("monitor using config['ethereum'] node") node_url = config['ethereum']['url'] self.eth = JsonRPCClient(node_url, should_retry=True) def start(self): if not hasattr(self, '_startup_future'): self._startup_future = asyncio.get_event_loop().create_future() asyncio.get_event_loop().create_task(self._initialise()) asyncio.get_event_loop().call_later( INITIAL_WAIT_CALLBACK_TIME, lambda: asyncio.get_event_loop(). create_task(self.run_erc20_health_check())) return self._startup_future @log_unhandled_exceptions(logger=log) async def _initialise(self): # prepare databases self.pool = await prepare_database(handle_migration=False) await prepare_redis() self._startup_future.set_result(True) async def run_erc20_health_check(self): try: await self._run_erc20_health_check() except: log.exception("Error running health check") asyncio.get_event_loop().call_later( ERC20_CHECK_CALLBACK_TIME, lambda: asyncio.get_event_loop(). create_task(self.run_erc20_health_check())) @log_unhandled_exceptions(logger=log) async def _run_erc20_health_check(self): log.info("running erc20 health check") async with self.pool.acquire() as con: token_balances = await con.fetch("SELECT * FROM token_balances") bad = 0 requests = [] last_execute = 0 bulk = self.eth.bulk() for token in token_balances: contract_address = token['contract_address'] data = "0x70a08231000000000000000000000000" + token['eth_address'][ 2:] f = bulk.eth_call(to_address=contract_address, data=data) requests.append( (contract_address, token['eth_address'], f, token['value'])) if len(requests) >= last_execute + 500: await bulk.execute() bulk = self.eth.bulk() last_execute = len(requests) if len(requests) > last_execute: await bulk.execute() bad_data = {} for contract_address, eth_address, f, db_value in requests: if not f.done(): log.warning("future not done when checking erc20 cache") continue try: value = f.result() except: log.exception("error getting erc20 value {}:{}".format( contract_address, eth_address)) continue if parse_int(value) != parse_int(db_value): bad += 1 bad_data.setdefault(eth_address, set()).add(contract_address) if bad > 0: log.warning( "Found {}/{} bad ERC20 caches over {} addresses".format( bad, len(token_balances), len(bad_data))) for eth_address in bad_data: erc20_dispatcher.update_token_cache("*", eth_address) await asyncio.sleep(15) # don't overload things
def eth(self): if not hasattr(self, '_eth_jsonrpc_client'): self._eth_jsonrpc_client = JsonRPCClient( config['ethereum']['url'], connect_timeout=5.0, request_timeout=5.0) return self._eth_jsonrpc_client
async def test_tx_queue_error_propagation(self, *, ethminer, parity, push_client): """Tests that a long chain of txs depending on a single transaction propagate errors correctly""" # start 2nd parity server p2 = ParityServer(bootnodes=parity.dsn()['node']) e2 = EthMiner(jsonrpc_url=p2.dsn()['url'], debug=False) rpcclient = JsonRPCClient(p2.dsn()['url']) default_fees = DEFAULT_STARTGAS * DEFAULT_GASPRICE val = 100 * 10 ** 18 txs = [] # send funds to address1 f_tx_hash = await self.send_tx(FAUCET_PRIVATE_KEY, TEST_ADDRESS_1, val) await self.ensure_confirmed(f_tx_hash) # make sure the nodes are synchronized while True: bal = await rpcclient.eth_getBalance(TEST_ADDRESS_1) if bal == 0: await asyncio.sleep(1) else: break # make sure no blocks are mined ethminer.pause() e2.pause() addresses = [(TEST_ADDRESS_1, TEST_PRIVATE_KEY_1), (TEST_ADDRESS_2, TEST_PRIVATE_KEY_2), (TEST_ADDRESS_3, TEST_PRIVATE_KEY_3), (TEST_ADDRESS_4, TEST_PRIVATE_KEY_4), (TEST_ADDRESS_5, TEST_PRIVATE_KEY_5), (TEST_ADDRESS_6, TEST_PRIVATE_KEY_6)] async with self.pool.acquire() as con: for addr, pk in addresses: await con.fetch("INSERT INTO notification_registrations (service, registration_id, toshi_id, eth_address) VALUES ($1, $2, $3, $4)", 'gcm', "abc", addr, addr) # send a tx from outside the system first which wont be seen by # the system until after the transactions generated in the next block tx = await self.get_tx_skel(TEST_PRIVATE_KEY_1, FAUCET_ADDRESS, val - default_fees) tx = sign_transaction(tx, TEST_PRIVATE_KEY_1) await rpcclient.eth_sendRawTransaction(tx) # generate internal transactions for i in range(len(addresses) * 2): val = val - default_fees addr1, pk1 = addresses[0] addr2, pk2 = addresses[1] # send funds tx_hash = await self.send_tx(pk1, addr2, val) txs.append(tx_hash) # swap all the variables addresses = addresses[1:] + [addresses[0]] # make sure we got pns for all for i in range(len(addresses) * 2): await push_client.get() await push_client.get() # start mining again e2.start() await self.ensure_errors(*txs) # make sure we got error pns for all for i in range(len(addresses) * 2): await push_client.get() await push_client.get() # and the pn for the overwritten tx await push_client.get()
def __init__(self): extra_service_config() self.eth = JsonRPCClient(config['ethereum']['url'], should_retry=False) asyncio.get_event_loop().create_task(self._initialize())
async def test_jsonrpc_errors(self, *, parity, push_client, monitor, fung): creator_contract = await self.deploy_contract(ARTTOKEN_CONTRACT, "ArtTokenCreator", []) async with self.pool.acquire() as con: await con.execute( "INSERT INTO collectibles (contract_address, name, type, image_url_format_string) VALUES ($1, $2, $3, $4)", creator_contract.address, "Art Tokens", 2, "https://ipfs.node/{token_uri}") await self.faucet(TEST_ADDRESS, 10**18) # "mint" some tokens txhash = await creator_contract.createAsset.set_sender( TEST_PRIVATE_KEY)("ART1", 10, "dasdasdasdasdasdasdasd", TEST_ADDRESS) receipt = await self.eth.eth_getTransactionReceipt(txhash) arttokenaddr = "0x" + receipt['logs'][0]['topics'][1][-40:] arttoken1 = await Contract.from_source_code(ARTTOKEN_CONTRACT, "ArtToken", address=arttokenaddr, deploy=False) # force block check to clear out txs pre registration await asyncio.sleep(0.1) await monitor.block_check() await asyncio.sleep(0.1) async with self.pool.acquire() as con: self.assertEqual( await con.fetchval("SELECT count(*) FROM fungible_collectibles"), 1) # send an art token! await arttoken1.transfer.set_sender(TEST_PRIVATE_KEY)(TEST_ADDRESS_2, 1) await asyncio.sleep(0.1) await monitor.block_check() await asyncio.sleep(0.1) async with self.pool.acquire() as con: owner_balance = await con.fetchval( "SELECT balance FROM fungible_collectible_balances WHERE owner_address = $1 AND contract_address = $2", TEST_ADDRESS, arttoken1.address) receiver_balance = await con.fetchval( "SELECT balance FROM fungible_collectible_balances WHERE owner_address = $1 AND contract_address = $2", TEST_ADDRESS_2, arttoken1.address) self.assertEqual(owner_balance, hex(9)) self.assertEqual(receiver_balance, hex(1)) # break the fungible monitor's eth instance old_fung_eth = fung.eth fung.eth = JsonRPCClient(self.get_url('/fake_jsonrpc'), should_retry=False) await arttoken1.transfer.set_sender(TEST_PRIVATE_KEY)(TEST_ADDRESS_3, 1) for i in range(10): async with self.pool.acquire() as con: val = await con.fetchval( "SELECT blocknumber FROM last_blocknumber") blk = await con.fetchval( "SELECT last_block FROM fungible_collectibles") count = await con.fetchval( "SELECT count(*) FROM fungible_collectible_balances") self.assertEqual(count, 2) if i > 0: self.assertNotEqual(val, blk) await asyncio.sleep(1) fung.eth = old_fung_eth await asyncio.sleep(5) async with self.pool.acquire() as con: val = await con.fetchval("SELECT blocknumber FROM last_blocknumber" ) blk = await con.fetchval( "SELECT last_block FROM fungible_collectibles") count = await con.fetchval( "SELECT count(*) FROM fungible_collectible_balances") self.assertEqual(count, 3) if val - blk > 1: self.fail( "fungible last_block is not caught up with the current block after failures" )
async def __call__(self, *args, startgas=None, gasprice=20000000000, value=0, wait_for_confirmation=True): # TODO: figure out if we can validate args validated_args = [] for (type, name), arg in zip( self.contract.translator.function_data[self.name]['signature'], args): if type == 'address' and isinstance(arg, str): validated_args.append(data_decoder(arg)) elif (type.startswith("uint") or type.startswith("int")) and isinstance(arg, str): validated_args.append(int(arg, 16)) else: validated_args.append(arg) ethurl = get_url() ethclient = JsonRPCClient(ethurl) data = self.contract.translator.encode_function_call( self.name, validated_args) # TODO: figure out if there's a better way to tell if the function needs to be called via sendTransaction if self.is_constant: result = await ethclient.eth_call(from_address=self.from_address or '', to_address=self.contract.address, data=data) result = data_decoder(result) if result: decoded = self.contract.translator.decode_function_result( self.name, result) # decode string results decoded = [ val.decode('utf-8') if isinstance(val, bytes) and type == 'string' else val for val, type in zip( decoded, self.contract.translator.function_data[ self.name]['decode_types']) ] # return the single value if there is only a single return value if len(decoded) == 1: return decoded[0] return decoded return None else: if self.from_address is None: raise Exception( "Cannot call non-constant function without a sender") nonce = await ethclient.eth_getTransactionCount(self.from_address) balance = await ethclient.eth_getBalance(self.from_address) if startgas is None: startgas = await ethclient.eth_estimateGas( self.from_address, self.contract.address, data=data, nonce=nonce, value=value, gasprice=gasprice) if startgas == 50000000 or startgas is None: raise Exception( "Unable to estimate gas cost, possibly something wrong with the transaction arguments" ) if balance < (startgas * gasprice): raise Exception("Given account doesn't have enough funds") tx = Transaction(nonce, gasprice, startgas, self.contract.address, value, data, 0, 0, 0) tx.sign(self.from_key) tx_encoded = data_encoder(rlp.encode(tx, Transaction)) if self.return_raw_tx: return tx_encoded try: tx_hash = await ethclient.eth_sendRawTransaction(tx_encoded) except: print(balance, startgas * gasprice, startgas) raise # wait for the contract to be deployed if wait_for_confirmation: print("waiting on transaction: {}".format(tx_hash)) starttime = time.time() warnlevel = 0 while wait_for_confirmation: resp = await ethclient.eth_getTransactionByHash(tx_hash) if resp is None or resp['blockNumber'] is None: await asyncio.sleep(0.1) if resp is None and warnlevel == 0 and time.time( ) - starttime < 10: print( "WARNING: 10 seconds have passed and transaction is not showing as a pending transaction" ) warnlevel = 1 elif resp is None and warnlevel == 1 and time.time( ) - starttime < 60: print( "WARNING: 60 seconds have passed and transaction is not showing as a pending transaction" ) raise Exception( "Unexpected error waiting for transaction to complete" ) else: receipt = await ethclient.eth_getTransactionReceipt(tx_hash ) if 'status' in receipt and receipt['status'] != "0x1": raise Exception( "Transaction status returned {}".format( receipt['status'])) break # TODO: is it possible for non-const functions to have return types? return tx_hash
async def from_source_code(cls, sourcecode, contract_name, constructor_data=None, *, address=None, deployer_private_key=None, import_mappings=None, libraries=None, optimize=False, deploy=True, cwd=None, wait_for_confirmation=True): if deploy: ethurl = get_url() if address is None and deployer_private_key is None: raise TypeError( "requires either address or deployer_private_key") if address is None and not isinstance(constructor_data, (list, type(None))): raise TypeError( "must supply constructor_data as a list (hint: use [] if args should be empty)" ) args = ['solc', '--combined-json', 'bin,abi'] if libraries: args.extend([ '--libraries', ','.join(['{}:{}'.format(*library) for library in libraries]) ]) if optimize: args.append('--optimize') if import_mappings: args.extend([ "{}={}".format(path, mapping) for path, mapping in import_mappings ]) # check if sourcecode is actually a filename if cwd: filename = os.path.join(cwd, sourcecode) else: filename = sourcecode if os.path.exists(filename): args.append(filename) sourcecode = None else: filename = '<stdin>' process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) output, stderrdata = process.communicate(input=sourcecode) try: output = json_decode(output) except json.JSONDecodeError: if output and stderrdata: output += b'\n' + stderrdata elif stderrdata: output = stderrdata raise Exception("Failed to compile source: {}\n{}\n{}".format( filename, ' '.join(args), output.decode('utf-8'))) try: contract = output['contracts']['{}:{}'.format( filename, contract_name)] except KeyError: print(output) raise abi = json_decode(contract['abi']) # deploy contract translator = ContractTranslator(abi) # fix things that don't have a constructor if not deploy: return Contract(abi=abi, address=address, translator=translator) ethclient = JsonRPCClient(ethurl) if address is not None: # verify there is code at the given address for i in range(10): code = await ethclient.eth_getCode(address) if code == "0x": await asyncio.sleep(1) continue break else: raise Exception("No code found at given address") return Contract(abi=abi, address=address, translator=translator) try: bytecode = data_decoder(contract['bin']) except binascii.Error: print(contract['bin']) raise if constructor_data is not None: constructor_call = translator.encode_constructor_arguments( constructor_data) bytecode += constructor_call if isinstance(deployer_private_key, str): deployer_private_key = data_decoder(deployer_private_key) deployer_address = private_key_to_address(deployer_private_key) nonce = await ethclient.eth_getTransactionCount(deployer_address) balance = await ethclient.eth_getBalance(deployer_address) gasprice = 20000000000 value = 0 startgas = await ethclient.eth_estimateGas(deployer_address, '', data=bytecode, nonce=nonce, value=0, gasprice=gasprice) if balance < (startgas * gasprice): raise Exception("Given account doesn't have enough funds") tx = Transaction(nonce, gasprice, startgas, '', value, bytecode, 0, 0, 0) tx.sign(deployer_private_key) tx_encoded = data_encoder(rlp.encode(tx, Transaction)) contract_address = data_encoder(tx.creates) tx_hash = await ethclient.eth_sendRawTransaction(tx_encoded) # wait for the contract to be deployed while wait_for_confirmation: resp = await ethclient.eth_getTransactionByHash(tx_hash) if resp is None or resp['blockNumber'] is None: await asyncio.sleep(0.1) else: code = await ethclient.eth_getCode(contract_address) if code == '0x': raise Exception( "Failed to deploy contract: resulting address '{}' has no code" .format(contract_address)) break return Contract(abi=abi, address=contract_address, translator=translator, creation_tx_hash=tx_hash)
class BlockMonitor: def __init__(self): configure_logger(log) if 'monitor' in config: node_url = config['monitor']['url'] else: log.warning("monitor using config['ethereum'] node") node_url = config['ethereum']['url'] self.eth = JsonRPCClient(node_url, connect_timeout=5.0, request_timeout=10.0) # filter health processes depend on some of the calls failing on the first time # so we have a separate client to handle those self.filter_eth = JsonRPCClient(node_url, force_instance=True, connect_timeout=10.0, request_timeout=60.0) self._check_schedule = None self._poll_schedule = None self._sanity_check_schedule = None self._block_checking_process = None self._filter_poll_process = None self._sanity_check_process = None self._process_unconfirmed_transactions_process = None self._new_pending_transaction_filter_id = None self._last_saw_new_block = asyncio.get_event_loop().time() self._shutdown = False self._lastlog = 0 self._blocktimes = [] def start(self): if not hasattr(self, '_startup_future'): self._startup_future = asyncio.get_event_loop().create_future() asyncio.get_event_loop().create_task(self._initialise()) self._sanity_check_schedule = asyncio.get_event_loop().call_later( SANITY_CHECK_CALLBACK_TIME, self.run_sanity_check) return self._startup_future @log_unhandled_exceptions(logger=log) async def _initialise(self): # prepare databases self.pool = await prepare_database(handle_migration=False) await prepare_redis() async with self.pool.acquire() as con: # check for the last non stale block processed row = await con.fetchrow( "SELECT blocknumber FROM blocks WHERE stale = FALSE ORDER BY blocknumber DESC LIMIT 1" ) if row is None: # fall back on old last_blocknumber row = await con.fetchrow( "SELECT blocknumber FROM last_blocknumber") if row is None: # if there was no previous start, get the current block number # and start from there last_block_number = await self.eth.eth_blockNumber() async with self.pool.acquire() as con: await con.execute("INSERT INTO last_blocknumber VALUES ($1)", last_block_number) else: last_block_number = row['blocknumber'] self.last_block_number = last_block_number self._shutdown = False await self.register_filters() self.schedule_filter_poll() self._startup_future.set_result(True) async def register_filters(self): if not self._shutdown: await self.register_new_pending_transaction_filter() async def register_new_pending_transaction_filter(self): backoff = 0 while not self._shutdown: try: filter_id = await self.filter_eth.eth_newPendingTransactionFilter( ) log.info( "Listening for new pending transactions with filter id: {}" .format(filter_id)) self._new_pending_transaction_filter_id = filter_id self._last_saw_new_pending_transactions = asyncio.get_event_loop( ).time() return filter_id except: log.exception("Error registering for new pending transactions") if not self._shutdown: backoff = min(backoff + 1, 10) await asyncio.sleep(backoff) def schedule_block_check(self, delay=DEFAULT_BLOCK_CHECK_DELAY): if self._shutdown: return self._check_schedule = asyncio.get_event_loop().call_later( delay, self.run_block_check) def schedule_filter_poll(self, delay=DEFAULT_POLL_DELAY): if self._shutdown: return self._poll_schedule = asyncio.get_event_loop().call_later( delay, self.run_filter_poll) def run_filter_poll(self): if self._shutdown: return if self._filter_poll_process is not None and not self._filter_poll_process.done( ): log.debug("filter polling is already running") return self._filter_poll_process = asyncio.get_event_loop().create_task( self.filter_poll()) def run_block_check(self): if self._shutdown: return if self._block_checking_process is not None and not self._block_checking_process.done( ): log.debug("Block check is already running") return self._block_checking_process = asyncio.get_event_loop().create_task( self.block_check()) def run_process_unconfirmed_transactions(self): if self._shutdown: return if self._process_unconfirmed_transactions_process is not None and not self._process_unconfirmed_transactions_process.done( ): log.debug("Process unconfirmed transactions is already running") return self._process_unconfirmed_transactions_process = asyncio.get_event_loop( ).create_task(self.process_unconfirmed_transactions()) @log_unhandled_exceptions(logger=log) async def block_check(self): while not self._shutdown: try: block = await self.eth.eth_getBlockByNumber( self.last_block_number + 1) except: log.exception("Failed eth_getBlockByNumber call") break if block: manager_dispatcher.update_default_gas_price( self.last_block_number + 1) self._last_saw_new_block = asyncio.get_event_loop().time() processing_start_time = asyncio.get_event_loop().time() if self._lastlog + 300 < asyncio.get_event_loop().time(): self._lastlog = asyncio.get_event_loop().time() log.info("Processing block {}".format(block['number'])) if len(self._blocktimes) > 0: log.info( "Average processing time per last {} blocks: {}". format( len(self._blocktimes), sum(self._blocktimes) / len(self._blocktimes))) # check for reorg async with self.pool.acquire() as con: last_block = await con.fetchrow( "SELECT * FROM blocks WHERE blocknumber = $1", self.last_block_number) # if we don't have the previous block, do a quick sanity check to see if there's any blocks lower if last_block is None: async with self.pool.acquire() as con: last_block_number = await con.fetchval( "SELECT blocknumber FROM blocks " "WHERE blocknumber < $1 " "ORDER BY blocknumber DESC LIMIT 1", self.last_block_number) if last_block_number: log.warning( "found gap in blocks @ block number: #{}".format( last_block_number + 1)) # roll back to the last block number and sync up self.last_block_number = last_block_number continue else: # make sure hash of the last block is the same as the current hash's parent block if last_block['hash'] != block['parentHash']: # we have a reorg! success = await self.handle_reorg() if success: continue # if we didn't find a reorg point, continue on as normal to avoid # preventing the system from operating as a whole # check if we're reorging async with self.pool.acquire() as con: is_reorg = await con.fetchval( "SELECT 1 FROM blocks WHERE blocknumber = $1", self.last_block_number + 1) if block['logsBloom'] != "0x" + ("0" * 512): try: logs_list = await self.eth.eth_getLogs( fromBlock=block['number'], toBlock=block['number']) except: log.exception("failed eth_getLogs call") break logs = {} for _log in logs_list: if _log['transactionHash'] not in logs: logs[_log['transactionHash']] = [_log] else: logs[_log['transactionHash']].append(_log) else: logs_list = [] logs = {} process_tx_tasks = [] for tx in block['transactions']: # send notifications to sender and reciever if tx['hash'] in logs: tx['logs'] = logs[tx['hash']] process_tx_tasks.append( asyncio.get_event_loop().create_task( self.process_transaction(tx, is_reorg=is_reorg))) await asyncio.gather(*process_tx_tasks) if logs_list: # send notifications for anyone registered async with self.pool.acquire() as con: for event in logs_list: for topic in event['topics']: filters = await con.fetch( "SELECT * FROM filter_registrations WHERE contract_address = $1 AND topic_id = $2", event['address'], topic) for filter in filters: eth_dispatcher.send_filter_notification( filter['filter_id'], filter['topic'], event['data']) # update the latest block number, only if it is larger than the # current block number. block_number = parse_int(block['number']) if self.last_block_number < block_number: self.last_block_number = block_number async with self.pool.acquire() as con: await con.execute( "UPDATE last_blocknumber SET blocknumber = $1 " "WHERE blocknumber < $1", block_number) await con.execute( "INSERT INTO blocks (blocknumber, timestamp, hash, parent_hash) " "VALUES ($1, $2, $3, $4) " "ON CONFLICT (blocknumber) DO UPDATE " "SET timestamp = EXCLUDED.timestamp, hash = EXCLUDED.hash, " "parent_hash = EXCLUDED.parent_hash, stale = FALSE", block_number, parse_int(block['timestamp']) or int(time.time()), block['hash'], block['parentHash']) collectibles_dispatcher.notify_new_block(block_number) processing_end_time = asyncio.get_event_loop().time() self._blocktimes.append(processing_end_time - processing_start_time) if len(self._blocktimes) > 100: self._blocktimes = self._blocktimes[-100:] else: break self._block_checking_process = None @log_unhandled_exceptions(logger=log) async def filter_poll(self): # check for newly added erc20 tokens if not self._shutdown: async with self.pool.acquire() as con: rows = await con.fetch( "SELECT contract_address FROM tokens WHERE ready = FALSE AND custom = FALSE" ) if len(rows) > 0: total_registrations = await con.fetchval( "SELECT COUNT(*) FROM token_registrations") else: total_registrations = 0 for row in rows: log.info("Got new erc20 token: {}. updating {} registrations". format(row['contract_address'], total_registrations)) if len(rows) > 0: limit = 1000 for offset in range(0, total_registrations, limit): async with self.pool.acquire() as con: registrations = await con.fetch( "SELECT eth_address FROM token_registrations OFFSET $1 LIMIT $2", offset, limit) for row in rows: erc20_dispatcher.update_token_cache( row['contract_address'], *[r['eth_address'] for r in registrations]) async with self.pool.acquire() as con: await con.executemany( "UPDATE tokens SET ready = true WHERE contract_address = $1", [(r['contract_address'], ) for r in rows]) if not self._shutdown: if self._new_pending_transaction_filter_id is not None: # get the list of new pending transactions try: new_pending_transactions = await self.filter_eth.eth_getFilterChanges( self._new_pending_transaction_filter_id) # add any to the list of unprocessed transactions for tx_hash in new_pending_transactions: await self.redis.hsetnx( UNCONFIRMED_TRANSACTIONS_REDIS_KEY, tx_hash, int(asyncio.get_event_loop().time())) except JSONRPC_ERRORS: log.exception("WARNING: unable to connect to server") new_pending_transactions = None if new_pending_transactions is None: await self.register_filters() elif len(new_pending_transactions) > 0: self._last_saw_new_pending_transactions = asyncio.get_event_loop( ).time() else: # make sure the filter timeout period hasn't passed time_since_last_pending_transaction = int( asyncio.get_event_loop().time() - self._last_saw_new_pending_transactions) if time_since_last_pending_transaction > FILTER_TIMEOUT: log.warning( "Haven't seen any new pending transactions for {} seconds" .format(time_since_last_pending_transaction)) await self.register_new_pending_transaction_filter() if await self.redis.hlen(UNCONFIRMED_TRANSACTIONS_REDIS_KEY ) > 0: self.run_process_unconfirmed_transactions() if not self._shutdown: # no need to run this if the block checking process is still running if self._block_checking_process is None or self._block_checking_process.done( ): try: block_number = await self.filter_eth.eth_blockNumber() except JSONRPC_ERRORS: log.exception("Error getting current block number") block_number = 0 if block_number > self.last_block_number and not self._shutdown: self.schedule_block_check() self._filter_poll_process = None if not self._shutdown: self.schedule_filter_poll(1 if ( await self.redis.hlen(UNCONFIRMED_TRANSACTIONS_REDIS_KEY) > 0 ) else DEFAULT_POLL_DELAY) @log_unhandled_exceptions(logger=log) async def process_unconfirmed_transactions(self): if self._shutdown: return # go through all the unmatched transactions that have no match unmatched_transactions = await self.redis.hgetall( UNCONFIRMED_TRANSACTIONS_REDIS_KEY, encoding="utf-8") for tx_hash, created in unmatched_transactions.items(): age = asyncio.get_event_loop().time() - int(created) try: tx = await self.eth.eth_getTransactionByHash(tx_hash) except JSONRPC_ERRORS: log.exception("Error getting transaction") tx = None if tx is None: # if the tx has existed for 60 seconds and not found, assume it was # removed from the network before being accepted into a block if age >= 60: await self.redis.hdel(UNCONFIRMED_TRANSACTIONS_REDIS_KEY, tx_hash) else: await self.redis.hdel(UNCONFIRMED_TRANSACTIONS_REDIS_KEY, tx_hash) # check if the transaction has already been included in a block # and if so, ignore this notification as it will be picked up by # the confirmed block check and there's no need to send two # notifications about it if tx['blockNumber'] is not None: continue await self.process_transaction(tx) if self._shutdown: break self._process_unconfirmed_transactions_process = None @log_unhandled_exceptions(logger=log) async def process_transaction(self, transaction, is_reorg=False): to_address = transaction['to'] # make sure we use a valid encoding of "empty" for contract deployments if to_address is None: to_address = "0x" from_address = transaction['from'] async with self.pool.acquire() as con: # find if we have a record of this tx by checking the from address and nonce db_txs = await con.fetch( "SELECT * FROM transactions WHERE " "from_address = $1 AND nonce = $2", from_address, parse_int(transaction['nonce'])) if len(db_txs) > 1: # see if one has the same hash db_tx = await con.fetchrow( "SELECT * FROM transactions WHERE " "from_address = $1 AND nonce = $2 AND hash = $3 AND (status != 'error' OR status = 'new')", from_address, parse_int(transaction['nonce']), transaction['hash']) if db_tx is None: # find if there are any that aren't marked as error no_error = await con.fetch( "SELECT * FROM transactions WHERE " "from_address = $1 AND nonce = $2 AND hash != $3 AND (status != 'error' OR status = 'new')", from_address, parse_int(transaction['nonce']), transaction['hash']) if len(no_error) == 1: db_tx = no_error[0] elif len(no_error) != 0: log.warning( "Multiple transactions from '{}' exist with nonce '{}' in unknown state" ) elif len(db_txs) == 1: db_tx = db_txs[0] else: db_tx = None # if we have a previous transaction, do some checking to see what's going on # see if this is an overwritten transaction # if the status of the old tx was previously an error, we don't care about it # otherwise, we have to notify the interested parties of the overwrite if db_tx and db_tx['hash'] != transaction['hash'] and db_tx[ 'status'] != 'error': if db_tx['v'] is not None: log.warning("found overwritten transaction!") log.warning("tx from: {}".format(from_address)) log.warning("nonce: {}".format( parse_int(transaction['nonce']))) log.warning("old tx hash: {}".format(db_tx['hash'])) log.warning("new tx hash: {}".format(transaction['hash'])) manager_dispatcher.update_transaction(db_tx['transaction_id'], 'error') db_tx = None # if reorg, and the transaction is confirmed, just update which block it was included in if is_reorg and db_tx and db_tx['hash'] == transaction[ 'hash'] and db_tx['status'] == 'confirmed': if transaction['blockNumber'] is None: log.error( "Unexpectedly got unconfirmed transaction again after reorg. hash: {}" .format(db_tx['hash'])) # this shouldn't really happen. going to log and abort return db_tx['transaction_id'] new_blocknumber = parse_int(transaction['blockNumber']) if new_blocknumber != db_tx['blocknumber']: async with self.pool.acquire() as con: await con.execute( "UPDATE transactions SET blocknumber = $1 " "WHERE transaction_id = $2", new_blocknumber, db_tx['transaction_id']) return db_tx['transaction_id'] # check for erc20 transfers erc20_transfers = [] if transaction['blockNumber'] is not None and \ 'logs' in transaction and \ len(transaction['logs']) > 0: # find any logs with erc20 token related topics for _log in transaction['logs']: if len(_log['topics']) > 0: # Transfer(address,address,uint256) if _log['topics'][0] == TRANSFER_TOPIC: # make sure the log address is for one we're interested in is_known_token = await con.fetchval( "SELECT 1 FROM tokens WHERE contract_address = $1", _log['address']) if not is_known_token: continue if len(_log['topics']) == 3 and len( _log['data']) == 66: # standard erc20 structure erc20_from_address = decode_single( ('address', '', []), data_decoder(_log['topics'][1])) erc20_to_address = decode_single( ('address', '', []), data_decoder(_log['topics'][2])) erc20_value = decode_abi(['uint256'], data_decoder( _log['data']))[0] elif len(_log['topics']) == 1 and len( _log['data']) == 194: # non-indexed style Transfer events erc20_from_address, erc20_to_address, erc20_value = decode_abi( ['address', 'address', 'uint256'], data_decoder(_log['data'])) else: log.warning( 'Got invalid erc20 Transfer event in tx: {}' .format(transaction['hash'])) continue erc20_is_interesting = await con.fetchval( "SELECT 1 FROM token_registrations " "WHERE eth_address = $1 OR eth_address = $2", erc20_from_address, erc20_to_address) if erc20_is_interesting: erc20_transfers.append( (_log['address'], get_transaction_log_index(_log), erc20_from_address, erc20_to_address, hex(erc20_value), 'confirmed')) # special checks for WETH, since it's rarely 'Transfer'ed, but we # still need to update it elif (_log['topics'][0] == DEPOSIT_TOPIC or _log['topics'][0] == WITHDRAWAL_TOPIC ) and _log['address'] == WETH_CONTRACT_ADDRESS: eth_address = decode_single( ('address', '', []), data_decoder(_log['topics'][1])) erc20_is_interesting = await con.fetchval( "SELECT 1 FROM token_registrations " "WHERE eth_address = $1", eth_address) if erc20_is_interesting: erc20_value = decode_abi(['uint256'], data_decoder( _log['data']))[0] if _log['topics'][0] == DEPOSIT_TOPIC: erc20_to_address = eth_address erc20_from_address = "0x0000000000000000000000000000000000000000" else: erc20_to_address = "0x0000000000000000000000000000000000000000" erc20_from_address = eth_address erc20_transfers.append( (WETH_CONTRACT_ADDRESS, get_transaction_log_index(_log), erc20_from_address, erc20_to_address, hex(erc20_value), 'confirmed')) elif transaction['blockNumber'] is None and db_tx is None: # transaction is pending, attempt to guess if this is a token # transaction based off it's input if transaction['input']: data = transaction['input'] if (data.startswith("0xa9059cbb") and len(data) == 138) or (data.startswith("0x23b872dd") and len(data) == 202): token_value = hex(int(data[-64:], 16)) if data.startswith("0x23b872dd"): erc20_from_address = "0x" + data[34:74] erc20_to_address = "0x" + data[98:138] else: erc20_from_address = from_address erc20_to_address = "0x" + data[34:74] erc20_transfers.append( (to_address, 0, erc20_from_address, erc20_to_address, token_value, 'unconfirmed')) # special WETH handling elif data == '0xd0e30db0' and transaction[ 'to'] == WETH_CONTRACT_ADDRESS: erc20_transfers.append( (WETH_CONTRACT_ADDRESS, 0, "0x0000000000000000000000000000000000000000", transaction['from'], transaction['value'], 'unconfirmed')) elif data.startswith('0x2e1a7d4d') and len(data) == 74: token_value = hex(int(data[-64:], 16)) erc20_transfers.append( (WETH_CONTRACT_ADDRESS, 0, transaction['from'], "0x0000000000000000000000000000000000000000", token_value, 'unconfirmed')) if db_tx: is_interesting = True else: # find out if there is anyone interested in this transaction is_interesting = await con.fetchval( "SELECT 1 FROM notification_registrations " "WHERE eth_address = $1 OR eth_address = $2", to_address, from_address) if not is_interesting and len(erc20_transfers) > 0: for _, _, erc20_from_address, erc20_to_address, _, _ in erc20_transfers: is_interesting = await con.fetchval( "SELECT 1 FROM notification_registrations " "WHERE eth_address = $1 OR eth_address = $2", erc20_to_address, erc20_from_address) if is_interesting: break is_interesting = await con.fetchval( "SELECT 1 FROM token_registrations " "WHERE eth_address = $1 OR eth_address = $2", erc20_to_address, erc20_from_address) if is_interesting: break if not is_interesting: return if db_tx is None: # if so, add it to the database and trigger an update # add tx to database db_tx = await con.fetchrow( "INSERT INTO transactions " "(hash, from_address, to_address, nonce, " "value, gas, gas_price, " "data) " "VALUES ($1, $2, $3, $4, $5, $6, $7, $8) " "RETURNING transaction_id", transaction['hash'], from_address, to_address, parse_int(transaction['nonce']), hex(parse_int(transaction['value'])), hex(parse_int(transaction['gas'])), hex(parse_int(transaction['gasPrice'])), transaction['input']) for erc20_contract_address, transaction_log_index, erc20_from_address, erc20_to_address, erc20_value, erc20_status in erc20_transfers: is_interesting = await con.fetchval( "SELECT 1 FROM notification_registrations " "WHERE eth_address = $1 OR eth_address = $2", erc20_to_address, erc20_from_address) if not is_interesting: is_interesting = await con.fetchrow( "SELECT 1 FROM token_registrations " "WHERE eth_address = $1 OR eth_address = $2", erc20_to_address, erc20_from_address) if is_interesting: await con.execute( "INSERT INTO token_transactions " "(transaction_id, transaction_log_index, contract_address, from_address, to_address, value, status) " "VALUES ($1, $2, $3, $4, $5, $6, $7) " "ON CONFLICT (transaction_id, transaction_log_index) DO UPDATE " "SET from_address = EXCLUDED.from_address, to_address = EXCLUDED.to_address, value = EXCLUDED.value", db_tx['transaction_id'], transaction_log_index, erc20_contract_address, erc20_from_address, erc20_to_address, erc20_value, erc20_status) manager_dispatcher.update_transaction( db_tx['transaction_id'], 'confirmed' if transaction['blockNumber'] is not None else 'unconfirmed') return db_tx['transaction_id'] @log_unhandled_exceptions(logger=log) async def handle_reorg(self): log.info("REORG encounterd at block #{}".format( self.last_block_number)) blocknumber = self.last_block_number forked_at_blocknumber = None BLOCKS_PER_ITERATION = 10 while True: bulk = self.eth.bulk() for i in range(BLOCKS_PER_ITERATION): if blocknumber - i >= 0: bulk.eth_getBlockByNumber(blocknumber - i, with_transactions=True) node_results = await bulk.execute() async with self.pool.acquire() as con: db_results = await con.fetch( "SELECT * FROM blocks WHERE blocknumber <= $1 ORDER BY blocknumber DESC LIMIT $2", blocknumber, BLOCKS_PER_ITERATION) while node_results: node_block = node_results[0] db_block = None while db_results: db_block = db_results[0] if parse_int( node_block['number']) != db_block['blocknumber']: log.error( "Got out of order blocks when handling reorg: expected: {}, got: {}" .format(parse_int(node_block['number']), db_block['blocknumber'])) db_results = db_results[1:] else: break if db_block is None: # we don't know about any more blocks, so we can just reorg the whole thing! break if node_block['hash'] == db_block['hash']: log.info("FORK found at block #{}".format( db_block['blocknumber'])) forked_at_blocknumber = db_block['blocknumber'] break log.info("Mismatched block #{}. old: {}, new: {}".format( db_block['blocknumber'], db_block['hash'], node_block['hash'])) node_results = node_results[1:] db_results = db_results[1:] if forked_at_blocknumber is not None: break blocknumber = blocknumber - BLOCKS_PER_ITERATION # if the blocknumber goes too low, abort finding the reorg if blocknumber <= 0 or blocknumber < self.last_block_number - 1000: log.error("UNABLE TO FIND FORK POINT FOR REORG") return False if forked_at_blocknumber is None: log.error( "Error: unexpectedly broke from reorg point finding loop") return False async with self.pool.acquire() as con: # mark blocks as stale await con.execute( "UPDATE blocks SET stale = TRUE WHERE blocknumber > $1", forked_at_blocknumber) # revert collectible's last block numbers await con.execute( "UPDATE collectibles SET last_block = $1 WHERE last_block > $1", forked_at_blocknumber - 1) self.last_block_number = forked_at_blocknumber return True def run_sanity_check(self): self._sanity_check_process = asyncio.get_event_loop().create_task( self.sanity_check()) @log_unhandled_exceptions(logger=log) async def sanity_check(self): if self._shutdown: return # check that filter ids are set to something if self._new_pending_transaction_filter_id is None: await self.register_new_pending_transaction_filter() # check that poll callback is set and not in the past if self._poll_schedule is None: log.warning("Filter poll schedule is None!") self.schedule_filter_poll() elif self._filter_poll_process is not None: pass else: if self._poll_schedule._when < self._poll_schedule._loop.time(): log.warning("Filter poll schedule is in the past!") self.schedule_filter_poll() # make sure there was a block somewhat recently ok = True time_since_last_new_block = int(asyncio.get_event_loop().time() - self._last_saw_new_block) if time_since_last_new_block > NEW_BLOCK_TIMEOUT: log.warning("Haven't seen any new blocks for {} seconds".format( time_since_last_new_block)) ok = False self._sanity_check_schedule = asyncio.get_event_loop().call_later( SANITY_CHECK_CALLBACK_TIME, self.run_sanity_check) if ok: await self.redis.setex("monitor_sanity_check_ok", SANITY_CHECK_CALLBACK_TIME * 2, "OK") self._sanity_check_process = None @property def redis(self): return get_redis_connection() async def shutdown(self): self._shutdown = True try: await self.filter_eth.close() except: pass if self._check_schedule: self._check_schedule.cancel() if self._poll_schedule: self._poll_schedule.cancel() if self._sanity_check_schedule: self._sanity_check_schedule.cancel() # let the current iteration of each process finish if running if self._block_checking_process: await self._block_checking_process if self._filter_poll_process: await self._filter_poll_process if self._sanity_check_process: await self._sanity_check_process if self._process_unconfirmed_transactions_process: await self._process_unconfirmed_transactions_process self._startup_future = None
async def update_default_gas_price(self, blocknumber): client = AsyncHTTPClient() fast_wei = None standard_wei = None safelow_wei = None eth_gasprice = None # only needed on mainnet if config['ethereum']['network_id'] == '1': try: resp = await client.fetch( "https://ethgasstation.info/json/ethgasAPI.json") rval = json_decode(resp.body) if 'fast' not in rval: log.error( "Unexpected results from EthGasStation: {}".format( resp.body)) elif not isinstance(rval['fast'], (int, float)): log.error( "Unexpected 'average' gas price returned by EthGasStation: {}" .format(rval['fast'])) else: gwei_x1000 = int(rval['fast'] * 100) fast_wei = gwei_x1000 * 1000000 if 'average' not in rval: log.error( "Unexpected results from EthGasStation: {}".format( resp.body)) elif not isinstance(rval['average'], (int, float)): log.error( "Unexpected 'average' gas price returned by EthGasStation: {}" .format(rval['average'])) else: gwei_x1000 = int(rval['average'] * 100) standard_wei = gwei_x1000 * 1000000 if 'safeLow' not in rval: log.error( "Unexpected results from EthGasStation: {}".format( resp.body)) elif not isinstance(rval['safeLow'], (int, float)): log.error( "Unexpected 'safeLow' gas price returned by EthGasStation: {}" .format(rval['safeLow'])) else: gwei_x1000 = int(rval['safeLow'] * 100) safelow_wei = gwei_x1000 * 1000000 # sanity check the values, if safelow is greater than standard # then use the safe low as standard + an extra gwei of padding if safelow_wei > standard_wei: standard_wei = safelow_wei + 1000000000 safelow_wei = hex(safelow_wei) standard_wei = hex(standard_wei) fast_wei = hex(fast_wei) await self.redis.mset('gas_station_safelow_gas_price', safelow_wei, 'gas_station_standard_gas_price', standard_wei, 'gas_station_fast_gas_price', fast_wei) except: log.exception( "Error updating default gas price from EthGasStation") try: # use the monitor url if available if 'monitor' in config: node_url = config['monitor']['url'] else: log.warning("monitor using config['ethereum'] node") node_url = config['ethereum']['url'] eth = JsonRPCClient(node_url, connect_timeout=5.0, request_timeout=10.0) eth_gasprice = await eth.eth_gasPrice() eth_gasprice = hex(eth_gasprice) except: log.exception("Error updating default gas price from eth node") # in case the eth gas station check failed, fall back on node gas price # if the node gas price is higher than the previous eth gas station fast gas price if fast_wei is None and eth_gasprice is not None: old_fast_wei = parse_int( await self.redis.get('gas_station_fast_gas_price')) if old_fast_wei is None or parse_int(eth_gasprice) > old_fast_wei: await self.redis.set('gas_station_fast_gas_price', eth_gasprice) async with self.db: await self.db.execute( "INSERT INTO gas_price_history " "(timestamp, blocknumber, gas_station_fast, gas_station_standard, gas_station_safelow, eth_gasprice) " "VALUES ($1, $2, $3, $4, $5, $6) " "ON CONFLICT (timestamp) DO NOTHING", int(time.time()), blocknumber, fast_wei, standard_wei, safelow_wei, eth_gasprice) await self.db.commit()