def deferred_blame_resolver(self): print("DEBUG: Started deferred_blame_resolver process.") database_connector = address_reuse.db.Database(TEMP_DB_FILENAME) blockchain_reader = address_reuse.blockchain_reader.LocalBlockchainRPCReader( database_connector) block_processor = address_reuse.block_processor.BlockProcessor( blockchain_reader, database_connector) subscriptions = address_reuse.data_subscription.BlockDataSubscriber( database=database_connector, next_block_needed=0) subscriptions.add_subscription( address_reuse.data_subscription.DataProducer. BLOCK_RELAYED_BY_CACHED_IN_DB) subscriptions.add_subscription( address_reuse.data_subscription.DataProducer. BLOCK_ADDRESS_REUSE_WITH_DEFERRED_BLAME) ready = subscriptions.are_producers_ready() self.assertFalse(ready) for i in range(0, 171): print( "DEBUG: deferred_blame_resolver @ height %d. May sleep now..." % i) subscriptions.do_sleep_until_producers_ready() print( "DEBUG: deferred_blame_resolver @ height %d. Produers are now ready." % i) block_processor.process_block_after_deferred_blaming(i) subscriptions.increment_next_block_needed()
def main(): purpose_to_elapsed = {} for cache_all_wallet_addresses in [True, False]: for cache_locally_for_all_input_addresses in [True, False]: block_processor = get_new_processor(cache_all_wallet_addresses, cache_locally_for_all_input_addresses) purpose = ('cache_all=%s, cache_input=%s' % (str(cache_all_wallet_addresses), str(cache_locally_for_all_input_addresses))) timer = address_reuse.time_debug.Timer(purpose) for block_height in range(150000, 150100): block_processor.process_block_after_deferred_blaming( block_height) elapsed = timer.stop() purpose_to_elapsed[purpose] = elapsed for purpose, elapsed in purpose_to_elapsed: print("%s: %s" % (purpose, str(elapsed)))
def deferred_blame_resolver(self): print("DEBUG: Started deferred_blame_resolver process.") database_connector = address_reuse.db.Database(TEMP_DB_FILENAME) blockchain_reader = address_reuse.blockchain_reader.LocalBlockchainRPCReader(database_connector) block_processor = address_reuse.block_processor.BlockProcessor(blockchain_reader, database_connector) subscriptions = address_reuse.data_subscription.BlockDataSubscriber( database=database_connector, next_block_needed=0 ) subscriptions.add_subscription(address_reuse.data_subscription.DataProducer.BLOCK_RELAYED_BY_CACHED_IN_DB) subscriptions.add_subscription( address_reuse.data_subscription.DataProducer.BLOCK_ADDRESS_REUSE_WITH_DEFERRED_BLAME ) ready = subscriptions.are_producers_ready() self.assertFalse(ready) for i in range(0, 171): print("DEBUG: deferred_blame_resolver @ height %d. May sleep now..." % i) subscriptions.do_sleep_until_producers_ready() print("DEBUG: deferred_blame_resolver @ height %d. Produers are now ready." % i) block_processor.process_block_after_deferred_blaming(i) subscriptions.increment_next_block_needed()
def main(): """Main function.""" my_pid = os.getpid() #Determine max number of blocks to process. -1 blocks = infinity config = address_reuse.config.Config( blockchain_mode=address_reuse.config.BlockchainMode.BITCOIND_RPC) blocks_remaining = config.MAX_NUM_BLOCKS_TO_PROCESS_PER_RUN db = address_reuse.db.Database( blockchain_mode=address_reuse.config.BlockchainMode.BITCOIND_RPC) coord_db = None if ENABLE_MULTITHREADING: coord_db = address_reuse.db.BlameResolverCoordinationDatabase() dead_blocks = coord_db.get_list_of_block_heights_with_possibly_crashed_workers() if len(dead_blocks) > 0: print ("Error: The following blocks were claimed by workers that " "appear to have crashed, and must be manually reclaimed. " "This can be done with the `repair_unclaim_dead_blocks_for_" "deferred_records.py` script.") for block in dead_blocks: print "\t%d" % int(block) sys.exit() current_height_iterated = 0 deque_of_claimed_blocks = None if ENABLE_MULTITHREADING: lowest_height_with_def_records = db.get_lowest_block_height_with_deferred_records() deque_of_claimed_blocks = coord_db.get_list_of_next_block_heights_available( starting_height=lowest_height_with_def_records, num_to_claim=NUM_CONSECUTIVE_BLOCKS_TO_CLAIM) try: current_height_iterated = deque_of_claimed_blocks.popleft() except IndexError: current_height_iterated = None else: current_height_iterated = db.get_lowest_block_height_with_deferred_records() if current_height_iterated is None: print("No deferred records found in database. TODO: Implement data " "subscription.") sys.exit() if HIGHEST_BLOCK_HEIGHT_TO_PROCESS is None: #process until we hit the current blockchain height api_reader = address_reuse.blockchain_reader.ThrottledBlockchainReader(db) max_blockchain_height = int(api_reader.get_current_blockchain_block_height()) api_reader = None #Done with API lookups :> else: max_blockchain_height = HIGHEST_BLOCK_HEIGHT_TO_PROCESS blockchain_reader = address_reuse.blockchain_reader.LocalBlockchainRPCReader(db) benchmarker = address_reuse.benchmark.block_reader_benchmark.Benchmark() last_completed_height = None try: while (current_height_iterated < max_blockchain_height and blocks_remaining): print(("DEBUG: update_deferred_blame_records.py: max block height to " "process is %d, last block processed in db is %d, %d " "remaining blocks to process in this run.") % (max_blockchain_height, current_height_iterated, blocks_remaining)) block_processor = address_reuse.block_processor.BlockProcessor( blockchain_reader, db) block_processor.process_block_after_deferred_blaming( current_height_iterated, benchmarker) #Log successful processing of this block address_reuse.logger.log_status(('Processed deferred blame for block ' '%d. (PID %d)') % (current_height_iterated, my_pid)) if ENABLE_MULTITHREADING: #record completion of this block height coord_db.mark_block_complete(current_height_iterated) print("Completed processing of block at height %d. (PID %d)" % (current_height_iterated, my_pid)) #Get next block height to process if ENABLE_MULTITHREADING: last_completed_height = current_height_iterated try: current_height_iterated = deque_of_claimed_blocks.popleft() except IndexError: #claim moar blocks deque_of_claimed_blocks = coord_db.get_list_of_next_block_heights_available( starting_height=current_height_iterated + 1, num_to_claim=NUM_CONSECUTIVE_BLOCKS_TO_CLAIM) if current_height_iterated > last_completed_height + 1: msg = (('Deferred blame processor skipping from block %d to ' '%d. (PID %d)') % (last_completed_height, current_height_iterated, my_pid)) address_reuse.logger.log_status(msg) else: current_height_iterated = current_height_iterated + 1 except Exception: traceback.print_exc() finally: #rollback claim on all blocks we've claimed so they aren't frozen if ENABLE_MULTITHREADING and isinstance(current_height_iterated, int): assert coord_db is not None for height in deque_of_claimed_blocks: if height != last_completed_height: coord_db.unclaim_block_height(height) if current_height_iterated != last_completed_height: coord_db.unclaim_block_height(current_height_iterated) #whether it finishes normally or is interrupted by ^C, print stats # before exiting benchmarker.stop() benchmarker.print_stats()