def sync_blockchain(bc_config, last_block=None, expected_snapshots={}, **virtualchain_args): """ synchronize state with the blockchain. Return True on success Return False if we're supposed to stop indexing Abort on error """ impl = sys.modules[__name__] if virtualchain.get_implementation() is not None: impl = None db_filename = virtualchain.get_db_filename(impl=impl) new_db = TalosPolicyDB(db_filename, expected_snapshots=expected_snapshots, read_only=False) try: if last_block is None: last_block = _get_newest_block(bc_config) rc = virtualchain.sync_virtualchain( bc_config, last_block, new_db, expected_snapshots=expected_snapshots) finally: new_db.close() return rc
def sync_blockchain(bt_opts, last_block, expected_snapshots={}, **virtualchain_args): """ synchronize state with the blockchain. Return True on success Return False if we're supposed to stop indexing Abort on error """ # make this usable even if we haven't explicitly configured virtualchain impl = sys.modules[__name__] if virtualchain.get_implementation() is not None: impl = None log.info("Synchronizing database up to block %s" % last_block) db_filename = virtualchain.get_db_filename(impl=impl) # NOTE: this is the only place where a read-write handle should be created, # since this is the only place where the db should be modified. new_db = BlockstackDB.borrow_readwrite_instance( db_filename, last_block, expected_snapshots=expected_snapshots) rc = virtualchain.sync_virtualchain(bt_opts, last_block, new_db, expected_snapshots=expected_snapshots, **virtualchain_args) BlockstackDB.release_readwrite_instance(new_db, last_block) return rc
def sync_blockchain( working_dir, bt_opts, last_block, server_state, expected_snapshots={}, **virtualchain_args ): """ synchronize state with the blockchain. Return True on success Return False if we're supposed to stop indexing Abort on error """ subdomain_index = server_state['subdomains'] atlas_state = server_state['atlas'] # make this usable even if we haven't explicitly configured virtualchain impl = sys.modules[__name__] log.info("Synchronizing database {} up to block {}".format(working_dir, last_block)) # NOTE: this is the only place where a read-write handle should be created, # since this is the only place where the db should be modified. new_db = BlockstackDB.borrow_readwrite_instance(working_dir, last_block, expected_snapshots=expected_snapshots) # propagate runtime state to virtualchain callbacks new_db.subdomain_index = subdomain_index new_db.atlas_state = atlas_state rc = virtualchain.sync_virtualchain(bt_opts, last_block, new_db, expected_snapshots=expected_snapshots, **virtualchain_args) BlockstackDB.release_readwrite_instance(new_db, last_block) return rc
def sync_virtualchain_upcall(zonefilemanage_opts, need_db_refresh): """ Upcall from the test scenario to synchronize virtualchain """ bitcoind = bitcoin_regtest_connect(bitcoin_regtest_opts()) height = bitcoind.getblockcount() db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW) testlib.set_state_engine(db) if need_db_refresh: pass old_lastblock = db.lastblock log.debug("Sync virtualchain up to %s " % height) virtualchain.sync_virtualchain(bitcoin_regtest_opts(), height, db)
def run_scenario(scenario, config_file): """ Run a test scenario: * set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain * seed it with the initial values in the wallet * set the initial consensus hash * run the scenario method * run the check method """ # use mock bitcoind virtualchain.setup_virtualchain( blockstore_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind) # set up blockstore # NOTE: utxo_opts encodes the mock-bitcoind options blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure( config_file=config_file, interactive=False) # override multiprocessing options to ensure single-process behavior utxo_opts['multiprocessing_num_procs'] = 1 utxo_opts['multiprocessing_num_blocks'] = 64 blockstored.set_bitcoin_opts(bitcoin_opts) blockstored.set_utxo_opts(utxo_opts) db = blockstored.get_state_engine() bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts) sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain( utxo_opts, bitcoind.getblockcount(), db) mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts) working_dir = virtualchain.get_working_dir() # set up test environment testlib.set_utxo_client(mock_utxo) testlib.set_bitcoind(bitcoind) testlib.set_state_engine(db) test_env = { "sync_virtualchain_upcall": sync_virtualchain_upcall, "working_dir": working_dir } # sync initial utxos testlib.next_block(**test_env) # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario(scenario.wallets, **test_env) except Exception, e: log.exception(e) traceback.print_exc() log.error("Failed to run scenario '%s'" % scenario.__name__) return False
def sync_blockchain( bt_opts, last_block ): """ synchronize state with the blockchain. build up the next blockstore_db """ global blockstore_db, blockstore_db_lock log.info("Synchronizing database up to block %s" % last_block) db_filename = virtualchain.get_db_filename() new_db = BlockstoreDB( db_filename ) virtualchain.sync_virtualchain( bt_opts, last_block, new_db ) # refresh blockstore_db_lock.acquire() del blockstore_db blockstore_db = new_db blockstore_db_lock.release()
def sync_blockchain(bt_opts, last_block): """ synchronize state with the blockchain. build up the next blockstore_db """ global blockstore_db, blockstore_db_lock log.info("Synchronizing database up to block %s" % last_block) db_filename = virtualchain.get_db_filename() new_db = BlockstoreDB(db_filename) virtualchain.sync_virtualchain(bt_opts, last_block, new_db) # refresh blockstore_db_lock.acquire() del blockstore_db blockstore_db = new_db blockstore_db_lock.release()
def run_indexer(): """ Continuously reindex the blockchain, but as a subprocess. """ # set up this process signal.signal( signal.SIGINT, sigint_handler_indexer ) bitcoind_opts = get_bitcoin_opts() _, last_block_id = get_index_range() blockstore_state_engine = get_state_engine() while True: time.sleep( REINDEX_FREQUENCY ) virtualchain.sync_virtualchain( bitcoind_opts, last_block_id, blockstore_state_engine ) _, last_block_id = get_index_range() return
def run_scenario( scenario, config_file ): """ Run a test scenario: * set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain * seed it with the initial values in the wallet * set the initial consensus hash * run the scenario method * run the check method """ # use mock bitcoind virtualchain.setup_virtualchain( blockstack_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind ) # set up blockstack # NOTE: utxo_opts encodes the mock-bitcoind options blockstack_opts, bitcoin_opts, utxo_opts, dht_opts = blockstack.lib.configure( config_file=config_file, interactive=False ) # override multiprocessing options to ensure single-process behavior utxo_opts['multiprocessing_num_procs'] = 1 utxo_opts['multiprocessing_num_blocks'] = 64 blockstackd.set_bitcoin_opts( bitcoin_opts ) blockstackd.set_utxo_opts( utxo_opts ) db = blockstackd.get_state_engine() bitcoind = mock_bitcoind.connect_mock_bitcoind( utxo_opts ) sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain( utxo_opts, bitcoind.getblockcount(), db ) mock_utxo = blockstack.lib.connect_utxo_provider( utxo_opts ) working_dir = virtualchain.get_working_dir() # set up test environment testlib.set_utxo_client( mock_utxo ) testlib.set_utxo_opts( utxo_opts ) testlib.set_bitcoind( bitcoind ) testlib.set_state_engine( db ) test_env = { "sync_virtualchain_upcall": sync_virtualchain_upcall, "working_dir": working_dir } # sync initial utxos testlib.next_block( **test_env ) # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario( scenario.wallets, **test_env ) except Exception, e: log.exception(e) traceback.print_exc() log.error("Failed to run scenario '%s'" % scenario.__name__) return False
def sync_blockchain( bt_opts, last_block, expected_snapshots={}, **virtualchain_args ): """ synchronize state with the blockchain. Return True on success Return False if we're supposed to stop indexing Abort on error """ # make this usable even if we haven't explicitly configured virtualchain impl = sys.modules[__name__] if virtualchain.get_implementation() is not None: impl = None log.info("Synchronizing database up to block %s" % last_block) db_filename = virtualchain.get_db_filename(impl=impl) # NOTE: this is the only place where a read-write handle should be created, # since this is the only place where the db should be modified. new_db = BlockstackDB.borrow_readwrite_instance( db_filename, last_block, expected_snapshots=expected_snapshots ) rc = virtualchain.sync_virtualchain( bt_opts, last_block, new_db, expected_snapshots=expected_snapshots, **virtualchain_args ) BlockstackDB.release_readwrite_instance( new_db, last_block ) return rc
"bitcoind_user": "******", "bitcoind_passwd": "talos", "bitcoind_server": "127.0.0.1", "bitcoind_p2p_port": 18444, "bitcoind_spv_path": "./tmp.dat" } """ conf={"bitcoind_port":13001, "bitcoind_user":"******", "bitcoind_passwd":"13000", "bitcoind_server":"13.93.113.195", "bitcoind_p2p_port":13000, "bitcoind_spv_path":"./tmp.dat"} """ """ conf={"bitcoind_port":8332, "bitcoind_user":"******", "bitcoind_passwd":"7876aeeebedebbc156b7dd69f9865af0fd8db6bc2afa6dc9eb64061ce9af91cc", "bitcoind_server":"127.0.0.1", "bitcoind_p2p_port":8333, "bitcoind_spv_path":"./tmp.dat"} """ # virtualchain.connect_bitcoind(conf) engine = StateEngine(util.MAGIC_BYTES, util.OPCODES, util.OPCODE_FIELDS, impl=chain) virtualchain.sync_virtualchain(conf, 879, engine)
def run_zonefilemanage(): working_dir = "/tmp/zonefilemanage" # errase prior state if os.path.exists(working_dir): log.debug("Remove %s " % working_dir) shutil.rmtree(working_dir) if not os.path.exists(working_dir): os.makedirs(working_dir) # export to test os.environ["VIRTUALCHAIN_WORKING_DIR"] = working_dir # set up bitcoind bitcoind_regtest_reset() virtualchain_working_dir = os.environ["VIRTUALCHAIN_WORKING_DIR"] spv_header_path = os.path.join(virtualchain_working_dir, "spv_headers.dat") virtualchain.setup_virtualchain(state_engine) # db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW) log.info("Connect to the bitcoind ") # Start the pinger pinger = Pinger() pinger.start() bitcoind = bitcoin_regtest_connect(bitcoin_regtest_opts()) if is_main_worker(): log.info("fill up the default wallet") # set up the default payment wallet default_payment_wallet = MultisigWallet(2, '5JYAj69z2GuFAZHrkhRuBKoCmKh6GcPXgcw9pbH8e8J2pu2RU9z', '5Kfg4xkZ1gGN5ozgDZ37Mn3EH9pXSuWZnQt1pzax4cLax8PetNs', '5JXB7rNxZa8yQtpuKtwy1nWUUTgdDEYTDmaEqQvKKC8HCWs64bL') # load wallets bitcoion_regtest_fill_wallets(wallets, default_payment_wallet=default_payment_wallet) else: # Watch out for wallets for wallet in wallets: testnet_wif = wallet.privkey if not testnet_wif.startswith("c"): testnet_wif = virtualchain.BitcoinPrivateKey(testnet_wif).to_wif() bitcoind.importprivkey(testnet_wif, "") addr = virtualchain.BitcoinPublicKey(wallet.pubkey_hex).address() log.info("Watch out for %s" % (addr)) for wallet in wallets: addr = get_wallet_addr(wallet) unspents = bitcoind.listunspent(0, 200000, [addr]) SATOSHIS_PER_COIN = 10 ** 8 value = sum([int(round(s["amount"] * SATOSHIS_PER_COIN)) for s in unspents]) print >> sys.stderr, "Address %s loaded with %s satoshis" % (addr, value) db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW) # Kill the pid for use this port return_code, output = commands.getstatusoutput("netstat -apn | grep %s | grep python| awk '{print $7}'" % RPC_SERVER_PORT) if 'python' in output: import re pattern = re.compile("(\d+)/python") match = pattern.search(output) if match: port = match.group(1) rc = os.system("kill -9 %s" % port) if rc != 0: log.exception("force kill failed") os.abort() # Start up the rpc server server = VoteServer() server.start() set_global_server(server) while True: height = bitcoind.getblockcount() log.info("Sync virtualchain up to %s " % height) virtualchain.sync_virtualchain(bitcoin_regtest_opts(), height, db) # wait for the next block deadline = time.time() + REINDEX_FREQUENCY while time.time() < deadline: try: time.sleep(1) except: break
def run_scenario(scenario, config_file): """ Run a test scenario: * set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain * seed it with the initial values in the wallet * set the initial consensus hash * run the scenario method * run the check method """ mock_bitcoind_save_path = "/tmp/mock_bitcoind.dat" if os.path.exists(mock_bitcoind_save_path): try: os.unlink(mock_bitcoind_save_path) except: pass # use mock bitcoind worker_env = { # use mock_bitcoind to connect to bitcoind (but it has to import it in order to use it) "VIRTUALCHAIN_MOD_CONNECT_BLOCKCHAIN": mock_bitcoind.__file__, "MOCK_BITCOIND_SAVE_PATH": mock_bitcoind_save_path, "BLOCKSTORE_TEST": "1" } if os.environ.get("PYTHONPATH", None) is not None: worker_env["PYTHONPATH"] = os.environ["PYTHONPATH"] virtualchain.setup_virtualchain( blockstore_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind, index_worker_env=worker_env) # set up blockstore # NOTE: utxo_opts encodes the mock-bitcoind options blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure( config_file=config_file, interactive=False) # override multiprocessing options to ensure single-process behavior utxo_opts['multiprocessing_num_procs'] = 1 utxo_opts['multiprocessing_num_blocks'] = 10 # pass along extra arguments utxo_opts['save_file'] = mock_bitcoind_save_path # save headers as well utxo_opts['spv_headers_path'] = mock_bitcoind_save_path + ".spvheaders" with open(utxo_opts['spv_headers_path'], "w") as f: # write out "initial" headers, up to the first block empty_header = ("00" * 81).decode('hex') for i in xrange(0, blockstore.FIRST_BLOCK_MAINNET): f.write(empty_header) blockstored.set_bitcoin_opts(bitcoin_opts) blockstored.set_utxo_opts(utxo_opts) db = blockstored.get_state_engine() bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts) sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain( utxo_opts, bitcoind.getblockcount(), db) mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts) working_dir = virtualchain.get_working_dir() # set up test environment testlib.set_utxo_opts(utxo_opts) testlib.set_utxo_client(mock_utxo) testlib.set_bitcoind(bitcoind) testlib.set_state_engine(db) test_env = { "sync_virtualchain_upcall": sync_virtualchain_upcall, "working_dir": working_dir, "bitcoind": bitcoind, "bitcoind_save_path": mock_bitcoind_save_path } # sync initial utxos testlib.next_block(**test_env) try: os.unlink(mock_bitcoind_save_path) except: pass # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario(scenario.wallets, **test_env) except Exception, e: log.exception(e) traceback.print_exc() log.error("Failed to run scenario '%s'" % scenario.__name__) return False
# count blocks as given (needs to be by ref) block_counter = {'count': virtualchain.get_first_block_id() } # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario( scenario.wallets, bitcoind=bitcoind, block_counter=block_counter ) except Exception, e: log.exception(e) log.error("Failed to run scenario '%s'" % scenario.__name__) return False # one more, just to cap it off testlib.next_block( bitcoind=bitcoind, block_counter=block_counter ) # crawl the mock blockchain virtualchain.sync_virtualchain( utxo_opts, block_counter['count'], db ) # run the checks on the database try: rc = scenario.check( db ) return rc except Exception, e: log.exception(e) log.error("Failed to run tests '%s'" % scenario.__name__) return False return rc if __name__ == "__main__":
def run_server( foreground=False): """ Run the blockstored RPC server, optionally in the foreground. """ signal.signal( signal.SIGINT, sigint_handler_server ) bitcoin_opts = get_bitcoin_opts() tac_file = get_tacfile_path() log_file = get_logfile_path() pid_file = get_pidfile_path() start_block, current_block = get_index_range() indexer_command = "%s indexer" % sys.argv[0] if foreground: command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file) else: command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file, log_file, tac_file) if start_block != current_block: # bring us up to speed log.info("Synchronizing with blockchain, up to %s" % current_block ) blockstore_state_engine = get_state_engine() virtualchain.sync_virtualchain( bitcoin_opts, current_block, blockstore_state_engine ) try: # fork the server blockstored = subprocess.Popen( command, shell=True, preexec_fn=os.setsid) # fork the indexer indexer = subprocess.Popen( indexer_command, shell=True ) log.info('Blockstored successfully started') # wait for it to die blockstored.wait() # stop our indexing thread os.kill( indexer.pid, signal.SIGINT ) indexer.wait() return blockstored.returncode except IndexError, ie: traceback.print_exc() # indicates that we don't have the latest block log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \ "Please verify that your bitcoin provider has processd up to" + \ "to block %s.\n" % (START_BLOCK) + \ " Example: bitcoin-cli getblockcount" ) try: os.killpg(blockstored.pid, signal.SIGTERM) except: pass exit(1)
def run_scenario(scenario, config_file): """ Run a test scenario: * set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain * seed it with the initial values in the wallet * set the initial consensus hash * run the scenario method * run the check method """ mock_bitcoind_save_path = "/tmp/mock_bitcoind.dat" if os.path.exists(mock_bitcoind_save_path): try: os.unlink(mock_bitcoind_save_path) except: pass # use mock bitcoind worker_env = { # use mock_bitcoind to connect to bitcoind (but it has to import it in order to use it) "VIRTUALCHAIN_MOD_CONNECT_BLOCKCHAIN": mock_bitcoind.__file__, "MOCK_BITCOIND_SAVE_PATH": mock_bitcoind_save_path, "BLOCKSTORE_TEST": "1", } if os.environ.get("PYTHONPATH", None) is not None: worker_env["PYTHONPATH"] = os.environ["PYTHONPATH"] virtualchain.setup_virtualchain( blockstore_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind, index_worker_env=worker_env, ) # set up blockstore # NOTE: utxo_opts encodes the mock-bitcoind options blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure( config_file=config_file, interactive=False ) # override multiprocessing options to ensure single-process behavior utxo_opts["multiprocessing_num_procs"] = 1 utxo_opts["multiprocessing_num_blocks"] = 10 # pass along extra arguments utxo_opts["save_file"] = mock_bitcoind_save_path # save headers as well utxo_opts["spv_headers_path"] = mock_bitcoind_save_path + ".spvheaders" with open(utxo_opts["spv_headers_path"], "w") as f: # write out "initial" headers, up to the first block empty_header = ("00" * 81).decode("hex") for i in xrange(0, blockstore.FIRST_BLOCK_MAINNET): f.write(empty_header) blockstored.set_bitcoin_opts(bitcoin_opts) blockstored.set_utxo_opts(utxo_opts) db = blockstored.get_state_engine() bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts) sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain(utxo_opts, bitcoind.getblockcount(), db) mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts) working_dir = virtualchain.get_working_dir() # set up test environment testlib.set_utxo_opts(utxo_opts) testlib.set_utxo_client(mock_utxo) testlib.set_bitcoind(bitcoind) testlib.set_state_engine(db) test_env = { "sync_virtualchain_upcall": sync_virtualchain_upcall, "working_dir": working_dir, "bitcoind": bitcoind, "bitcoind_save_path": mock_bitcoind_save_path, } # sync initial utxos testlib.next_block(**test_env) try: os.unlink(mock_bitcoind_save_path) except: pass # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario(scenario.wallets, **test_env) except Exception, e: log.exception(e) traceback.print_exc() log.error("Failed to run scenario '%s'" % scenario.__name__) return False
# foreground api_server_command = ('twistd --pidfile=%s -noy %s' % (pid_file, tac_file)).split() # start API server blockstored = subprocess.Popen( api_server_command, shell=False) set_indexing( False ) if start_block != current_block: # bring us up to speed set_indexing( True ) blockstore_state_engine = get_state_engine() virtualchain.sync_virtualchain( bt_opts, current_block, blockstore_state_engine ) set_indexing( False ) # fork the indexer if foreground: indexer = subprocess.Popen( indexer_command, shell=False ) else: indexer = subprocess.Popen( indexer_command, shell=False, stdout=logfile, stderr=logfile ) indexer_pid = indexer.pid # wait for the API server to die (we kill it with `blockstored stop`) blockstored.wait() # stop our indexer subprocess