def run(self): logger.debug('Starting API Status Poller.') global current_api_status_code, current_api_status_response_json db = database.get_connection(read_only=True, integrity_check=False) while self.stop_event.is_set() != True: try: # Check that backend is running, communicable, and caught up with the blockchain. # Check that the database has caught up with bitcoind. if time.time() - self.last_database_check > 10 * 60: # Ten minutes since last check. if not config.FORCE: code = 11 logger.debug('Checking backend state.') check_backend_state() code = 12 logger.debug('Checking database state.') check_database_state(db, backend.getblockcount()) self.last_database_check = time.time() except (BackendError, DatabaseError) as e: exception_name = e.__class__.__name__ exception_text = str(e) logger.debug("API Status Poller: %s", exception_text) jsonrpc_response = jsonrpc.exceptions.JSONRPCServerError(message=exception_name, data=exception_text) current_api_status_code = code current_api_status_response_json = jsonrpc_response.json.encode() else: current_api_status_code = None current_api_status_response_json = None time.sleep(config.BACKEND_POLL_INTERVAL)
def run(self): logger.debug('Starting API Status Poller.') global current_api_status_code, current_api_status_response_json db = database.get_connection(read_only=True, integrity_check=False) while self.stop_event.is_set() != True: try: # Check that bitcoind is running, communicable, and caught up with the blockchain. # Check that the database has caught up with bitcoind. if time.time( ) - self.last_database_check > 10 * 60: # Ten minutes since last check. code = 11 logger.debug('Checking backend state.') check.backend_state() code = 12 logger.debug('Checking database state.') check.database_state(db, backend.getblockcount()) self.last_database_check = time.time() except (check.BackendError, check.DatabaseError) as e: exception_name = e.__class__.__name__ exception_text = str(e) logger.debug("API Status Poller: %s", exception_text) jsonrpc_response = jsonrpc.exceptions.JSONRPCServerError( message=exception_name, data=exception_text) current_api_status_code = code current_api_status_response_json = jsonrpc_response.json.encode( ) else: current_api_status_code = None current_api_status_response_json = None time.sleep(config.BACKEND_POLL_INTERVAL)
def get_running_info(): latestBlockIndex = backend.getblockcount() try: check_database_state(db, latestBlockIndex) except DatabaseError: caught_up = False else: caught_up = True try: cursor = db.cursor() blocks = list(cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (util.CURRENT_BLOCK_INDEX, ))) assert len(blocks) == 1 last_block = blocks[0] cursor.close() except: last_block = None try: last_message = util.last_message(db) except: last_message = None return { 'db_caught_up': caught_up, 'bitcoin_block_count': latestBlockIndex, 'last_block': last_block, 'last_message_index': last_message['message_index'] if last_message else -1, 'running_testnet': config.TESTNET, 'running_testcoin': config.TESTCOIN, 'version_major': config.VERSION_MAJOR, 'version_minor': config.VERSION_MINOR, 'version_revision': config.VERSION_REVISION }
def handle_healthz(): msg, code = 'Healthy', 200 try: latestBlockIndex = backend.getblockcount() check_database_state(self.db, latestBlockIndex) except DatabaseError: msg, code = 'Unhealthy', 503 return flask.Response(msg, code, mimetype='text/plain')
def check_backend_state(): """Checks blocktime of last block to see if {} Core is running behind.""".format(config.BTC_NAME) block_count = backend.getblockcount() block_hash = backend.getblockhash(block_count) cblock = backend.getblock(block_hash) time_behind = time.time() - cblock.nTime # TODO: Block times are not very reliable. if time_behind > 60 * 60 * 2: # Two hours. raise BackendError('Bitcoind is running about {} hours behind.'.format(round(time_behind / 3600))) logger.debug('Backend state check passed.')
def get_running_info(): latestBlockIndex = backend.getblockcount() try: check_database_state(self.db, latestBlockIndex) except DatabaseError: caught_up = False else: caught_up = True try: cursor = self.db.cursor() blocks = list(cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (util.CURRENT_BLOCK_INDEX, ))) assert len(blocks) == 1 last_block = blocks[0] cursor.close() except: last_block = None try: last_message = util.last_message(self.db) except: last_message = None try: indexd_blocks_behind = backend.getindexblocksbehind() except: indexd_blocks_behind = latestBlockIndex if latestBlockIndex > 0 else 999999 indexd_caught_up = indexd_blocks_behind <= 1 server_ready = caught_up and indexd_caught_up return { 'server_ready': server_ready, 'db_caught_up': caught_up, 'bitcoin_block_count': latestBlockIndex, 'last_block': last_block, 'indexd_caught_up': indexd_caught_up, 'indexd_blocks_behind': indexd_blocks_behind, 'last_message_index': last_message['message_index'] if last_message else -1, 'api_limit_rows': config.API_LIMIT_ROWS, 'running_testnet': config.TESTNET, 'running_regtest': config.REGTEST, 'running_testcoin': config.TESTCOIN, 'version_major': config.VERSION_MAJOR, 'version_minor': config.VERSION_MINOR, 'version_revision': config.VERSION_REVISION }
def connect_to_backend(): if not config.FORCE: logger.info('Connecting to backend.') backend.getblockcount()
def follow(db): # Check software version. check.software_version() # Initialise. initialise(db) # Get index of last block. if util.CURRENT_BLOCK_INDEX == 0: logger.warning('New database.') block_index = config.BLOCK_FIRST else: block_index = util.CURRENT_BLOCK_INDEX + 1 # Check database version. try: check.database_version(db) except check.DatabaseVersionError as e: logger.info(str(e)) # no need to reparse or rollback a new database if block_index != config.BLOCK_FIRST: reparse(db, block_index=e.reparse_block_index, quiet=False) database.update_version(db) logger.info('Resuming parsing.') # Get index of last transaction. tx_index = get_next_tx_index(db) not_supported = {} # No false positives. Use a dict to allow for O(1) lookups not_supported_sorted = collections.deque() # ^ Entries in form of (block_index, tx_hash), oldest first. Allows for easy removal of past, unncessary entries mempool_initialised = False backend.init_mempool_cache() cursor = db.cursor() # a reorg can happen without the block count increasing, or even for that # matter, with the block count decreasing. This should only delay # processing of the new blocks a bit. while True: starttime = time.time() # Get block count. # If the backend is unreachable and `config.FORCE` is set, just sleep # and try again repeatedly. try: block_count = backend.getblockcount() except (ConnectionRefusedError, http.client.CannotSendRequest, backend.addrindex.BackendRPCError) as e: if config.FORCE: time.sleep(config.BACKEND_POLL_INTERVAL) continue else: raise e # Get new blocks. if block_index <= block_count: # Backwards check for incorrect blocks due to chain reorganisation, and stop when a common parent is found. current_index = block_index requires_rollback = False while True: if current_index == config.BLOCK_FIRST: break logger.debug('Checking that block {} is not an orphan.'.format(current_index)) # Backend parent hash. current_hash = backend.getblockhash(current_index) current_cblock = backend.getblock(current_hash) backend_parent = bitcoinlib.core.b2lx(current_cblock.hashPrevBlock) # DB parent hash. blocks = list(cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (current_index - 1,))) if len(blocks) != 1: # For empty DB. break db_parent = blocks[0]['block_hash'] # Compare. assert type(db_parent) == str assert type(backend_parent) == str if db_parent == backend_parent: break else: current_index -= 1 requires_rollback = True # Rollback for reorganisation. if requires_rollback: # Record reorganisation. logger.warning('Blockchain reorganisation at block {}.'.format(current_index)) log.message(db, block_index, 'reorg', None, {'block_index': current_index}) # Rollback the DB. reparse(db, block_index=current_index-1, quiet=True) block_index = current_index tx_index = get_next_tx_index(db) continue # Check version. (Don’t add any blocks to the database while # running an out‐of‐date client!) check.software_version() # Get and parse transactions in this block (atomically). block_hash = backend.getblockhash(current_index) block = backend.getblock(block_hash) previous_block_hash = bitcoinlib.core.b2lx(block.hashPrevBlock) block_time = block.nTime txhash_list = backend.get_txhash_list(block) raw_transactions = backend.getrawtransaction_batch(txhash_list) with db: util.CURRENT_BLOCK_INDEX = block_index # List the block. cursor.execute('''INSERT INTO blocks( block_index, block_hash, block_time, previous_block_hash, difficulty) VALUES(?,?,?,?,?)''', (block_index, block_hash, block_time, previous_block_hash, block.difficulty) ) # List the transactions in the block. for tx_hash in txhash_list: tx_hex = raw_transactions[tx_hash] tx_index = list_tx(db, block_hash, block_index, block_time, tx_hash, tx_index, tx_hex) # Parse the transactions in the block. parse_block(db, block_index, block_time) # When newly caught up, check for conservation of assets. if block_index == block_count: if config.CHECK_ASSET_CONSERVATION: check.asset_conservation(db) # Remove any non‐supported transactions older than ten blocks. while len(not_supported_sorted) and not_supported_sorted[0][0] <= block_index - 10: tx_h = not_supported_sorted.popleft()[1] del not_supported[tx_h] logger.info('Block: %s (%ss)'%(str(block_index), "{:.2f}".format(time.time() - starttime, 3))) # Increment block index. block_count = backend.getblockcount() block_index += 1 else: # First mempool fill for session? if mempool_initialised: logger.debug('Updating mempool.') else: logger.debug('Initialising mempool.') # Get old mempool. old_mempool = list(cursor.execute('''SELECT * FROM mempool''')) old_mempool_hashes = [message['tx_hash'] for message in old_mempool] # Fake values for fake block. curr_time = int(time.time()) mempool_tx_index = tx_index # For each transaction in Bitcoin Core mempool, if it’s new, create # a fake block, a fake transaction, capture the generated messages, # and then save those messages. # Every transaction in mempool is parsed independently. (DB is rolled back after each one.) mempool = [] for tx_hash in backend.getrawmempool(): # If already in mempool, copy to new one. if tx_hash in old_mempool_hashes: for message in old_mempool: if message['tx_hash'] == tx_hash: mempool.append((tx_hash, message)) # If already skipped, skip it again. elif tx_hash not in not_supported: # Else: list, parse and save it. try: with db: # List the fake block. cursor.execute('''INSERT INTO blocks( block_index, block_hash, block_time) VALUES(?,?,?)''', (config.MEMPOOL_BLOCK_INDEX, config.MEMPOOL_BLOCK_HASH, curr_time) ) # List transaction. try: # Sometimes the transactions can’t be found: `{'code': -5, 'message': 'No information available about transaction'} Is txindex enabled in Bitcoind?` mempool_tx_index = list_tx(db, None, block_index, curr_time, tx_hash, mempool_tx_index) except backend.addrindex.BackendRPCError: raise MempoolError # Parse transaction. cursor.execute('''SELECT * FROM transactions \ WHERE tx_hash = ?''', (tx_hash,)) transactions = list(cursor) if transactions: assert len(transactions) == 1 transaction = transactions[0] supported = parse_tx(db, transaction) if not supported: not_supported[tx_hash] = '' not_supported_sorted.append((block_index, tx_hash)) else: # If a transaction hasn’t been added to the # table `transactions`, then it’s not a # Counterparty transaction. not_supported[tx_hash] = '' not_supported_sorted.append((block_index, tx_hash)) raise MempoolError # Save transaction and side‐effects in memory. cursor.execute('''SELECT * FROM messages WHERE block_index = ?''', (config.MEMPOOL_BLOCK_INDEX,)) for message in list(cursor): mempool.append((tx_hash, message)) # Rollback. raise MempoolError except MempoolError: pass # Re‐write mempool messages to database. with db: cursor.execute('''DELETE FROM mempool''') for message in mempool: tx_hash, new_message = message new_message['tx_hash'] = tx_hash cursor.execute('''INSERT INTO mempool VALUES(:tx_hash, :command, :category, :bindings, :timestamp)''', (new_message)) # Wait mempool_initialised = True db.wal_checkpoint(mode=apsw.SQLITE_CHECKPOINT_PASSIVE) time.sleep(config.BACKEND_POLL_INTERVAL) cursor.close()
def follow(db): # Check software version. check.software_version() # Initialise. initialise(db) # Get index of last block. if util.CURRENT_BLOCK_INDEX == 0: logger.warning('New database.') block_index = config.BLOCK_FIRST else: block_index = util.CURRENT_BLOCK_INDEX + 1 # Check database version. try: check.database_version(db) except check.DatabaseVersionError as e: logger.info(str(e)) # no need to reparse or rollback a new database if block_index != config.BLOCK_FIRST: reparse(db, block_index=e.reparse_block_index, quiet=False) database.update_version(db) logger.info('Resuming parsing.') # Get index of last transaction. tx_index = get_next_tx_index(db) not_supported = { } # No false positives. Use a dict to allow for O(1) lookups not_supported_sorted = collections.deque() # ^ Entries in form of (block_index, tx_hash), oldest first. Allows for easy removal of past, unncessary entries mempool_initialised = False cursor = db.cursor() # a reorg can happen without the block count increasing, or even for that # matter, with the block count decreasing. This should only delay # processing of the new blocks a bit. while True: starttime = time.time() # Get block count. # If the backend is unreachable and `config.FORCE` is set, just sleep # and try again repeatedly. try: block_count = backend.getblockcount() except (ConnectionRefusedError, http.client.CannotSendRequest) as e: if config.FORCE: time.sleep(config.BACKEND_POLL_INTERVAL) continue else: raise e # Get new blocks. if block_index <= block_count: # Backwards check for incorrect blocks due to chain reorganisation, and stop when a common parent is found. current_index = block_index requires_rollback = False while True: if current_index == config.BLOCK_FIRST: break logger.debug('Checking that block {} is not an orphan.'.format( current_index)) # Backend parent hash. current_hash = backend.getblockhash(current_index) current_cblock = backend.getblock(current_hash) backend_parent = bitcoinlib.core.b2lx( current_cblock.hashPrevBlock) # DB parent hash. blocks = list( cursor.execute( '''SELECT * FROM blocks WHERE block_index = ?''', (current_index - 1, ))) if len(blocks) != 1: # For empty DB. break db_parent = blocks[0]['block_hash'] # Compare. assert type(db_parent) == str assert type(backend_parent) == str if db_parent == backend_parent: break else: current_index -= 1 requires_rollback = True # Rollback for reorganisation. if requires_rollback: # Record reorganisation. logger.warning('Blockchain reorganisation at block {}.'.format( current_index)) log.message(db, block_index, 'reorg', None, {'block_index': current_index}) # Rollback the DB. reparse(db, block_index=current_index - 1, quiet=True) block_index = current_index tx_index = get_next_tx_index(db) continue # Check version. (Don’t add any blocks to the database while # running an out‐of‐date client!) check.software_version() # Get and parse transactions in this block (atomically). block_hash = backend.getblockhash(current_index) block = backend.getblock(block_hash) previous_block_hash = bitcoinlib.core.b2lx(block.hashPrevBlock) block_time = block.nTime txhash_list = backend.get_txhash_list(block) with db: util.CURRENT_BLOCK_INDEX = block_index # List the block. cursor.execute( '''INSERT INTO blocks( block_index, block_hash, block_time, previous_block_hash, difficulty) VALUES(?,?,?,?,?)''', (block_index, block_hash, block_time, previous_block_hash, block.difficulty)) # List the transactions in the block. for tx_hash in txhash_list: # TODO: use rpc._batch to get all transactions with one RPC call tx_index = list_tx(db, block_hash, block_index, block_time, tx_hash, tx_index) # Parse the transactions in the block. parse_block(db, block_index, block_time) # When newly caught up, check for conservation of assets. if block_index == block_count: check.asset_conservation(db) # Remove any non‐supported transactions older than ten blocks. while len(not_supported_sorted ) and not_supported_sorted[0][0] <= block_index - 10: tx_h = not_supported_sorted.popleft()[1] del not_supported[tx_h] logger.info('Block: %s (%ss)' % (str(block_index), "{:.2f}".format( time.time() - starttime, 3))) # Increment block index. block_count = backend.getblockcount() block_index += 1 else: # First mempool fill for session? if mempool_initialised: logger.debug('Updating mempool.') else: logger.debug('Initialising mempool.') # Get old counterpartyd mempool. old_mempool = list(cursor.execute('''SELECT * FROM mempool''')) old_mempool_hashes = [ message['tx_hash'] for message in old_mempool ] # Fake values for fake block. curr_time = int(time.time()) mempool_tx_index = tx_index # For each transaction in Bitcoin Core mempool, if it’s new, create # a fake block, a fake transaction, capture the generated messages, # and then save those messages. # Every transaction in mempool is parsed independently. (DB is rolled back after each one.) mempool = [] for tx_hash in backend.getrawmempool(): # If already in counterpartyd mempool, copy to new one. if tx_hash in old_mempool_hashes: for message in old_mempool: if message['tx_hash'] == tx_hash: mempool.append((tx_hash, message)) # If already skipped, skip it again. elif tx_hash not in not_supported: # Else: list, parse and save it. try: with db: # List the fake block. cursor.execute( '''INSERT INTO blocks( block_index, block_hash, block_time) VALUES(?,?,?)''', (config.MEMPOOL_BLOCK_INDEX, config.MEMPOOL_BLOCK_HASH, curr_time)) # List transaction. try: # Sometimes the transactions can’t be found: `{'code': -5, 'message': 'No information available about transaction'} Is txindex enabled in Bitcoind?` mempool_tx_index = list_tx( db, None, block_index, curr_time, tx_hash, mempool_tx_index) except backend.addrindex.BackendRPCError: raise MempoolError # Parse transaction. cursor.execute( '''SELECT * FROM transactions \ WHERE tx_hash = ?''', (tx_hash, )) transactions = list(cursor) if transactions: assert len(transactions) == 1 transaction = transactions[0] supported = parse_tx(db, transaction) if not supported: not_supported[tx_hash] = '' not_supported_sorted.append( (block_index, tx_hash)) else: # If a transaction hasn’t been added to the # table `transactions`, then it’s not a # Counterparty transaction. not_supported[tx_hash] = '' not_supported_sorted.append( (block_index, tx_hash)) raise MempoolError # Save transaction and side‐effects in memory. cursor.execute( '''SELECT * FROM messages WHERE block_index = ?''', (config.MEMPOOL_BLOCK_INDEX, )) for message in list(cursor): mempool.append((tx_hash, message)) # Rollback. raise MempoolError except MempoolError: pass # Re‐write mempool messages to database. with db: cursor.execute('''DELETE FROM mempool''') for message in mempool: tx_hash, new_message = message new_message['tx_hash'] = tx_hash cursor.execute( '''INSERT INTO mempool VALUES(:tx_hash, :command, :category, :bindings, :timestamp)''', (new_message)) # Wait mempool_initialised = True db.wal_checkpoint(mode=apsw.SQLITE_CHECKPOINT_PASSIVE) time.sleep(config.BACKEND_POLL_INTERVAL) cursor.close()