def run_scenario(scenario, config_file): """ Run a test scenario: * set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain * seed it with the initial values in the wallet * set the initial consensus hash * run the scenario method * run the check method """ # use mock bitcoind virtualchain.setup_virtualchain( blockstore_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind) # set up blockstore # NOTE: utxo_opts encodes the mock-bitcoind options blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure( config_file=config_file, interactive=False) # override multiprocessing options to ensure single-process behavior utxo_opts['multiprocessing_num_procs'] = 1 utxo_opts['multiprocessing_num_blocks'] = 64 blockstored.set_bitcoin_opts(bitcoin_opts) blockstored.set_utxo_opts(utxo_opts) db = blockstored.get_state_engine() bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts) sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain( utxo_opts, bitcoind.getblockcount(), db) mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts) working_dir = virtualchain.get_working_dir() # set up test environment testlib.set_utxo_client(mock_utxo) testlib.set_bitcoind(bitcoind) testlib.set_state_engine(db) test_env = { "sync_virtualchain_upcall": sync_virtualchain_upcall, "working_dir": working_dir } # sync initial utxos testlib.next_block(**test_env) # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario(scenario.wallets, **test_env) except Exception, e: log.exception(e) traceback.print_exc() log.error("Failed to run scenario '%s'" % scenario.__name__) return False
def run_scenario( scenario, config_file ): """ Run a test scenario: * set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain * seed it with the initial values in the wallet * set the initial consensus hash * run the scenario method * run the check method """ # use mock bitcoind virtualchain.setup_virtualchain( blockstack_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind ) # set up blockstack # NOTE: utxo_opts encodes the mock-bitcoind options blockstack_opts, bitcoin_opts, utxo_opts, dht_opts = blockstack.lib.configure( config_file=config_file, interactive=False ) # override multiprocessing options to ensure single-process behavior utxo_opts['multiprocessing_num_procs'] = 1 utxo_opts['multiprocessing_num_blocks'] = 64 blockstackd.set_bitcoin_opts( bitcoin_opts ) blockstackd.set_utxo_opts( utxo_opts ) db = blockstackd.get_state_engine() bitcoind = mock_bitcoind.connect_mock_bitcoind( utxo_opts ) sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain( utxo_opts, bitcoind.getblockcount(), db ) mock_utxo = blockstack.lib.connect_utxo_provider( utxo_opts ) working_dir = virtualchain.get_working_dir() # set up test environment testlib.set_utxo_client( mock_utxo ) testlib.set_utxo_opts( utxo_opts ) testlib.set_bitcoind( bitcoind ) testlib.set_state_engine( db ) test_env = { "sync_virtualchain_upcall": sync_virtualchain_upcall, "working_dir": working_dir } # sync initial utxos testlib.next_block( **test_env ) # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario( scenario.wallets, **test_env ) except Exception, e: log.exception(e) traceback.print_exc() log.error("Failed to run scenario '%s'" % scenario.__name__) return False
def setup( return_parser=False ): """ Do one-time initialization. Call this to set up global state and set signal handlers. If return_parser is True, return a partially- setup argument parser to be populated with subparsers (i.e. as part of main()) Otherwise return None. """ global blockstore_opts global blockchain_client global blockchain_broadcaster global bitcoin_opts global utxo_opts global blockstore_opts global dht_opts # set up our implementation virtualchain.setup_virtualchain( blockstore_state_engine ) # acquire configuration, and store it globally blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = configure( interactive=True ) # merge in command-line bitcoind options config_file = virtualchain.get_config_filename() arg_bitcoin_opts = None argparser = None if return_parser: arg_bitcoin_opts, argparser = virtualchain.parse_bitcoind_args( return_parser=return_parser ) else: arg_bitcoin_opts = virtualchain.parse_bitcoind_args( return_parser=return_parser ) # command-line overrides config file for (k, v) in arg_bitcoin_opts.items(): bitcoin_opts[k] = v # store options set_bitcoin_opts( bitcoin_opts ) set_utxo_opts( utxo_opts ) if return_parser: return argparser else: return None
def setup( return_parser=False ): """ Do one-time initialization. Call this to set up global state and set signal handlers. If return_parser is True, return a partially- setup argument parser to be populated with subparsers (i.e. as part of main()) Otherwise return None. """ global blockchain_client global bitcoin_opts global chaincom_opts # set up our implementation virtualchain.setup_virtualchain( blockstore_state_engine ) # acquire configuration, and store it globally bitcoin_opts, chaincom_opts = configure( interactive=True ) # merge in command-line bitcoind options config_file = virtualchain.get_config_filename() arg_bitcoin_opts = None argparser = None if return_parser: arg_bitcoin_opts, argparser = virtualchain.parse_bitcoind_args( return_parser=return_parser ) else: arg_bitcoin_opts = virtualchain.parse_bitcoind_args( return_parser=return_parser ) # command-line overrides config file for (k, v) in arg_bitcoin_opts.items(): bitcoin_opts[k] = v # store options set_bitcoin_opts( bitcoin_opts ) set_chaincom_opts( chaincom_opts ) if return_parser: return argparser else: return None
def run_scenario( scenario, config_file ): """ Run a test scenario: * set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain * seed it with the initial values in the wallet * set the initial consensus hash * run the scenario method * run the check method """ # use mock bitcoind virtualchain.setup_virtualchain( blockstore_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind ) # set up blockstore # NOTE: utxo_opts encodes the mock-bitcoind options blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure( config_file=config_file, interactive=False ) # override multiprocessing options to ensure single-process behavior utxo_opts['multiprocessing_num_procs'] = 1 utxo_opts['multiprocessing_num_blocks'] = 64 blockstored.set_bitcoin_opts( bitcoin_opts ) blockstored.set_utxo_opts( utxo_opts ) db = blockstored.get_state_engine() bitcoind = mock_bitcoind.connect_mock_bitcoind( utxo_opts ) # count blocks as given (needs to be by ref) block_counter = {'count': virtualchain.get_first_block_id() } # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario( scenario.wallets, bitcoind=bitcoind, block_counter=block_counter ) except Exception, e: log.exception(e) log.error("Failed to run scenario '%s'" % scenario.__name__) return False
def rebuild_database( target_block_id, untrusted_db_path, working_db_path=None, resume_dir=None, start_block=None, expected_snapshots={} ): """ Given a target block ID and a path to an (untrusted) db, reconstruct it in a temporary directory by replaying all the nameops it contains. Optionally check that the snapshots in @expected_snapshots match up as we verify. @expected_snapshots maps str(block_id) to str(consensus hash) Return the consensus hash calculated at the target block. Return None on verification failure (i.e. we got a different consensus hash than one for the same block in expected_snapshots) """ # reconfigure the virtualchain to use a temporary directory, # so we don't interfere with this instance's primary database working_dir = None if resume_dir is None: working_dir = tempfile.mkdtemp( prefix='blockstack-verify-database-' ) else: working_dir = resume_dir blockstack_state_engine.working_dir = working_dir virtualchain.setup_virtualchain( impl=blockstack_state_engine ) if resume_dir is None: # not resuming start_block = virtualchain.get_first_block_id() else: # resuming old_start_block = start_block start_block = get_lastblock() if start_block is None: start_block = old_start_block log.debug( "Rebuilding database from %s to %s" % (start_block, target_block_id) ) # feed in operations, block by block, from the untrusted database untrusted_db = BlockstackDB( untrusted_db_path, DISPOSITION_RO ) # working db, to build up the operations in the untrusted db block-by-block working_db = None if working_db_path is None: working_db_path = virtualchain.get_db_filename() working_db = BlockstackDB( working_db_path, DISPOSITION_RW ) # map block ID to consensus hashes consensus_hashes = {} for block_id in xrange( start_block, target_block_id+1 ): untrusted_db.lastblock = block_id virtualchain_ops = block_to_virtualchain_ops( block_id, working_db, untrusted_db ) # feed ops to virtualchain to reconstruct the db at this block consensus_hash = working_db.process_block( block_id, virtualchain_ops ) log.debug("VERIFY CONSENSUS(%s): %s" % (block_id, consensus_hash)) consensus_hashes[block_id] = consensus_hash if block_id in expected_snapshots: if expected_snapshots[block_id] != consensus_hash: log.error("DATABASE IS NOT CONSISTENT AT %s: %s != %s" % (block_id, expected_snashots[block_id], consensus_hash)) return None # final consensus hash return consensus_hashes[ target_block_id ]
import virtualchain from virtualchain import StateEngine import test_chain as chain import util virtualchain.setup_virtualchain(impl=chain) conf = { "bitcoind_port": 18332, "bitcoind_user": "******", "bitcoind_passwd": "talos", "bitcoind_server": "127.0.0.1", "bitcoind_p2p_port": 18444, "bitcoind_spv_path": "./tmp.dat" } """ conf={"bitcoind_port":13001, "bitcoind_user":"******", "bitcoind_passwd":"13000", "bitcoind_server":"13.93.113.195", "bitcoind_p2p_port":13000, "bitcoind_spv_path":"./tmp.dat"} """ """ conf={"bitcoind_port":8332, "bitcoind_user":"******", "bitcoind_passwd":"7876aeeebedebbc156b7dd69f9865af0fd8db6bc2afa6dc9eb64061ce9af91cc", "bitcoind_server":"127.0.0.1", "bitcoind_p2p_port":8333, "bitcoind_spv_path":"./tmp.dat"}
def run_zonefilemanage(): working_dir = "/tmp/zonefilemanage" # errase prior state if os.path.exists(working_dir): log.debug("Remove %s " % working_dir) shutil.rmtree(working_dir) if not os.path.exists(working_dir): os.makedirs(working_dir) # export to test os.environ["VIRTUALCHAIN_WORKING_DIR"] = working_dir # set up bitcoind bitcoind_regtest_reset() virtualchain_working_dir = os.environ["VIRTUALCHAIN_WORKING_DIR"] spv_header_path = os.path.join(virtualchain_working_dir, "spv_headers.dat") virtualchain.setup_virtualchain(state_engine) # db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW) log.info("Connect to the bitcoind ") # Start the pinger pinger = Pinger() pinger.start() bitcoind = bitcoin_regtest_connect(bitcoin_regtest_opts()) if is_main_worker(): log.info("fill up the default wallet") # set up the default payment wallet default_payment_wallet = MultisigWallet(2, '5JYAj69z2GuFAZHrkhRuBKoCmKh6GcPXgcw9pbH8e8J2pu2RU9z', '5Kfg4xkZ1gGN5ozgDZ37Mn3EH9pXSuWZnQt1pzax4cLax8PetNs', '5JXB7rNxZa8yQtpuKtwy1nWUUTgdDEYTDmaEqQvKKC8HCWs64bL') # load wallets bitcoion_regtest_fill_wallets(wallets, default_payment_wallet=default_payment_wallet) else: # Watch out for wallets for wallet in wallets: testnet_wif = wallet.privkey if not testnet_wif.startswith("c"): testnet_wif = virtualchain.BitcoinPrivateKey(testnet_wif).to_wif() bitcoind.importprivkey(testnet_wif, "") addr = virtualchain.BitcoinPublicKey(wallet.pubkey_hex).address() log.info("Watch out for %s" % (addr)) for wallet in wallets: addr = get_wallet_addr(wallet) unspents = bitcoind.listunspent(0, 200000, [addr]) SATOSHIS_PER_COIN = 10 ** 8 value = sum([int(round(s["amount"] * SATOSHIS_PER_COIN)) for s in unspents]) print >> sys.stderr, "Address %s loaded with %s satoshis" % (addr, value) db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW) # Kill the pid for use this port return_code, output = commands.getstatusoutput("netstat -apn | grep %s | grep python| awk '{print $7}'" % RPC_SERVER_PORT) if 'python' in output: import re pattern = re.compile("(\d+)/python") match = pattern.search(output) if match: port = match.group(1) rc = os.system("kill -9 %s" % port) if rc != 0: log.exception("force kill failed") os.abort() # Start up the rpc server server = VoteServer() server.start() set_global_server(server) while True: height = bitcoind.getblockcount() log.info("Sync virtualchain up to %s " % height) virtualchain.sync_virtualchain(bitcoin_regtest_opts(), height, db) # wait for the next block deadline = time.time() + REINDEX_FREQUENCY while time.time() < deadline: try: time.sleep(1) except: break
def run_scenario(scenario, config_file): """ Run a test scenario: * set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain * seed it with the initial values in the wallet * set the initial consensus hash * run the scenario method * run the check method """ mock_bitcoind_save_path = "/tmp/mock_bitcoind.dat" if os.path.exists(mock_bitcoind_save_path): try: os.unlink(mock_bitcoind_save_path) except: pass # use mock bitcoind worker_env = { # use mock_bitcoind to connect to bitcoind (but it has to import it in order to use it) "VIRTUALCHAIN_MOD_CONNECT_BLOCKCHAIN": mock_bitcoind.__file__, "MOCK_BITCOIND_SAVE_PATH": mock_bitcoind_save_path, "BLOCKSTORE_TEST": "1" } if os.environ.get("PYTHONPATH", None) is not None: worker_env["PYTHONPATH"] = os.environ["PYTHONPATH"] virtualchain.setup_virtualchain( blockstore_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind, index_worker_env=worker_env) # set up blockstore # NOTE: utxo_opts encodes the mock-bitcoind options blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure( config_file=config_file, interactive=False) # override multiprocessing options to ensure single-process behavior utxo_opts['multiprocessing_num_procs'] = 1 utxo_opts['multiprocessing_num_blocks'] = 10 # pass along extra arguments utxo_opts['save_file'] = mock_bitcoind_save_path # save headers as well utxo_opts['spv_headers_path'] = mock_bitcoind_save_path + ".spvheaders" with open(utxo_opts['spv_headers_path'], "w") as f: # write out "initial" headers, up to the first block empty_header = ("00" * 81).decode('hex') for i in xrange(0, blockstore.FIRST_BLOCK_MAINNET): f.write(empty_header) blockstored.set_bitcoin_opts(bitcoin_opts) blockstored.set_utxo_opts(utxo_opts) db = blockstored.get_state_engine() bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts) sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain( utxo_opts, bitcoind.getblockcount(), db) mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts) working_dir = virtualchain.get_working_dir() # set up test environment testlib.set_utxo_opts(utxo_opts) testlib.set_utxo_client(mock_utxo) testlib.set_bitcoind(bitcoind) testlib.set_state_engine(db) test_env = { "sync_virtualchain_upcall": sync_virtualchain_upcall, "working_dir": working_dir, "bitcoind": bitcoind, "bitcoind_save_path": mock_bitcoind_save_path } # sync initial utxos testlib.next_block(**test_env) try: os.unlink(mock_bitcoind_save_path) except: pass # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario(scenario.wallets, **test_env) except Exception, e: log.exception(e) traceback.print_exc() log.error("Failed to run scenario '%s'" % scenario.__name__) return False
def run_scenario(scenario, config_file, client_config_file, interactive=False, blocktime=10): """ * set up the virtualchain to use mock UTXO provider and mock bitcoin blockchain * seed it with the intial value in the wallet * set the intial consensus hash * start the api server * run the scenario method * run the check method """ virtualchain_working_dir = os.environ["VIRTUALCHAIN_WORKING_DIR"] spv_header_path = os.path.join(virtualchain_working_dir, "spv_headers.dat") virtualchain.setup_virtualchain(state_engine) db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW) log.info("Connect to the bitcoind ") bitcoind = bitcoin_regtest_connect(bitcoin_regtest_opts()) working_dir = get_working_dir() utxo_opts = {} # Start the pinger pinger = Pinger() pinger.start() #set up the environment testlib.set_utxo_opts(utxo_opts) testlib.set_bitcoind(bitcoind) testlib.set_state_engine(db) test_env = { "sync_virtualchain_upcall": lambda: sync_virtualchain_upcall(zonefilemanage_opts=None, need_db_refresh=False), "next_block_upcall": bitcoin_regtest_next_block, "working_dir": working_dir, "bitcoind": bitcoind, "bitcoind_opts": bitcoin_regtest_opts(), "spv_header_path": spv_header_path } # Sync initial utxos testlib.next_block(**test_env) # Load the scenario into the mock blockchain and mock utxo provider try: rc = scenario.scenario(scenario.wallets, **test_env) except Exception, e: log.exception(e) traceback.print_exc()
def run_scenario(scenario, config_file): """ Run a test scenario: * set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain * seed it with the initial values in the wallet * set the initial consensus hash * run the scenario method * run the check method """ mock_bitcoind_save_path = "/tmp/mock_bitcoind.dat" if os.path.exists(mock_bitcoind_save_path): try: os.unlink(mock_bitcoind_save_path) except: pass # use mock bitcoind worker_env = { # use mock_bitcoind to connect to bitcoind (but it has to import it in order to use it) "VIRTUALCHAIN_MOD_CONNECT_BLOCKCHAIN": mock_bitcoind.__file__, "MOCK_BITCOIND_SAVE_PATH": mock_bitcoind_save_path, "BLOCKSTORE_TEST": "1", } if os.environ.get("PYTHONPATH", None) is not None: worker_env["PYTHONPATH"] = os.environ["PYTHONPATH"] virtualchain.setup_virtualchain( blockstore_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind, index_worker_env=worker_env, ) # set up blockstore # NOTE: utxo_opts encodes the mock-bitcoind options blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure( config_file=config_file, interactive=False ) # override multiprocessing options to ensure single-process behavior utxo_opts["multiprocessing_num_procs"] = 1 utxo_opts["multiprocessing_num_blocks"] = 10 # pass along extra arguments utxo_opts["save_file"] = mock_bitcoind_save_path # save headers as well utxo_opts["spv_headers_path"] = mock_bitcoind_save_path + ".spvheaders" with open(utxo_opts["spv_headers_path"], "w") as f: # write out "initial" headers, up to the first block empty_header = ("00" * 81).decode("hex") for i in xrange(0, blockstore.FIRST_BLOCK_MAINNET): f.write(empty_header) blockstored.set_bitcoin_opts(bitcoin_opts) blockstored.set_utxo_opts(utxo_opts) db = blockstored.get_state_engine() bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts) sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain(utxo_opts, bitcoind.getblockcount(), db) mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts) working_dir = virtualchain.get_working_dir() # set up test environment testlib.set_utxo_opts(utxo_opts) testlib.set_utxo_client(mock_utxo) testlib.set_bitcoind(bitcoind) testlib.set_state_engine(db) test_env = { "sync_virtualchain_upcall": sync_virtualchain_upcall, "working_dir": working_dir, "bitcoind": bitcoind, "bitcoind_save_path": mock_bitcoind_save_path, } # sync initial utxos testlib.next_block(**test_env) try: os.unlink(mock_bitcoind_save_path) except: pass # load the scenario into the mock blockchain and mock utxo provider try: scenario.scenario(scenario.wallets, **test_env) except Exception, e: log.exception(e) traceback.print_exc() log.error("Failed to run scenario '%s'" % scenario.__name__) return False