def sync_blockchain(bc_config,
last_block=None,
expected_snapshots={},
**virtualchain_args):
"""
synchronize state with the blockchain.
Return True on success
Return False if we're supposed to stop indexing
Abort on error
"""
impl = sys.modules[__name__]
if virtualchain.get_implementation() is not None:
impl = None
db_filename = virtualchain.get_db_filename(impl=impl)
new_db = TalosPolicyDB(db_filename,
expected_snapshots=expected_snapshots,
read_only=False)
try:
if last_block is None:
last_block = _get_newest_block(bc_config)
rc = virtualchain.sync_virtualchain(
bc_config,
last_block,
new_db,
expected_snapshots=expected_snapshots)
finally:
new_db.close()
return rc
def sync_blockchain(bt_opts,
last_block,
expected_snapshots={},
**virtualchain_args):
"""
synchronize state with the blockchain.
Return True on success
Return False if we're supposed to stop indexing
Abort on error
"""
# make this usable even if we haven't explicitly configured virtualchain
impl = sys.modules[__name__]
if virtualchain.get_implementation() is not None:
impl = None
log.info("Synchronizing database up to block %s" % last_block)
db_filename = virtualchain.get_db_filename(impl=impl)
# NOTE: this is the only place where a read-write handle should be created,
# since this is the only place where the db should be modified.
new_db = BlockstackDB.borrow_readwrite_instance(
db_filename, last_block, expected_snapshots=expected_snapshots)
rc = virtualchain.sync_virtualchain(bt_opts,
last_block,
new_db,
expected_snapshots=expected_snapshots,
**virtualchain_args)
BlockstackDB.release_readwrite_instance(new_db, last_block)
return rc
def sync_blockchain( working_dir, bt_opts, last_block, server_state, expected_snapshots={}, **virtualchain_args ):
"""
synchronize state with the blockchain.
Return True on success
Return False if we're supposed to stop indexing
Abort on error
"""
subdomain_index = server_state['subdomains']
atlas_state = server_state['atlas']
# make this usable even if we haven't explicitly configured virtualchain
impl = sys.modules[__name__]
log.info("Synchronizing database {} up to block {}".format(working_dir, last_block))
# NOTE: this is the only place where a read-write handle should be created,
# since this is the only place where the db should be modified.
new_db = BlockstackDB.borrow_readwrite_instance(working_dir, last_block, expected_snapshots=expected_snapshots)
# propagate runtime state to virtualchain callbacks
new_db.subdomain_index = subdomain_index
new_db.atlas_state = atlas_state
rc = virtualchain.sync_virtualchain(bt_opts, last_block, new_db, expected_snapshots=expected_snapshots, **virtualchain_args)
BlockstackDB.release_readwrite_instance(new_db, last_block)
return rc
def sync_virtualchain_upcall(zonefilemanage_opts, need_db_refresh):
"""
Upcall from the test scenario to synchronize virtualchain
"""
bitcoind = bitcoin_regtest_connect(bitcoin_regtest_opts())
height = bitcoind.getblockcount()
db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW)
testlib.set_state_engine(db)
if need_db_refresh:
pass
old_lastblock = db.lastblock
log.debug("Sync virtualchain up to %s " % height)
virtualchain.sync_virtualchain(bitcoin_regtest_opts(), height, db)
def run_scenario(scenario, config_file):
"""
Run a test scenario:
* set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain
* seed it with the initial values in the wallet
* set the initial consensus hash
* run the scenario method
* run the check method
"""
# use mock bitcoind
virtualchain.setup_virtualchain(
blockstore_state_engine,
bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind)
# set up blockstore
# NOTE: utxo_opts encodes the mock-bitcoind options
blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure(
config_file=config_file, interactive=False)
# override multiprocessing options to ensure single-process behavior
utxo_opts['multiprocessing_num_procs'] = 1
utxo_opts['multiprocessing_num_blocks'] = 64
blockstored.set_bitcoin_opts(bitcoin_opts)
blockstored.set_utxo_opts(utxo_opts)
db = blockstored.get_state_engine()
bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts)
sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain(
utxo_opts, bitcoind.getblockcount(), db)
mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts)
working_dir = virtualchain.get_working_dir()
# set up test environment
testlib.set_utxo_client(mock_utxo)
testlib.set_bitcoind(bitcoind)
testlib.set_state_engine(db)
test_env = {
"sync_virtualchain_upcall": sync_virtualchain_upcall,
"working_dir": working_dir
}
# sync initial utxos
testlib.next_block(**test_env)
# load the scenario into the mock blockchain and mock utxo provider
try:
scenario.scenario(scenario.wallets, **test_env)
except Exception, e:
log.exception(e)
traceback.print_exc()
log.error("Failed to run scenario '%s'" % scenario.__name__)
return False
def sync_blockchain( bt_opts, last_block ):
"""
synchronize state with the blockchain.
build up the next blockstore_db
"""
global blockstore_db, blockstore_db_lock
log.info("Synchronizing database up to block %s" % last_block)
db_filename = virtualchain.get_db_filename()
new_db = BlockstoreDB( db_filename )
virtualchain.sync_virtualchain( bt_opts, last_block, new_db )
# refresh
blockstore_db_lock.acquire()
del blockstore_db
blockstore_db = new_db
blockstore_db_lock.release()
def sync_blockchain(bt_opts, last_block):
"""
synchronize state with the blockchain.
build up the next blockstore_db
"""
global blockstore_db, blockstore_db_lock
log.info("Synchronizing database up to block %s" % last_block)
db_filename = virtualchain.get_db_filename()
new_db = BlockstoreDB(db_filename)
virtualchain.sync_virtualchain(bt_opts, last_block, new_db)
# refresh
blockstore_db_lock.acquire()
del blockstore_db
blockstore_db = new_db
blockstore_db_lock.release()
def run_indexer():
"""
Continuously reindex the blockchain, but as a subprocess.
"""
# set up this process
signal.signal( signal.SIGINT, sigint_handler_indexer )
bitcoind_opts = get_bitcoin_opts()
_, last_block_id = get_index_range()
blockstore_state_engine = get_state_engine()
while True:
time.sleep( REINDEX_FREQUENCY )
virtualchain.sync_virtualchain( bitcoind_opts, last_block_id, blockstore_state_engine )
_, last_block_id = get_index_range()
return
def run_scenario( scenario, config_file ):
"""
Run a test scenario:
* set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain
* seed it with the initial values in the wallet
* set the initial consensus hash
* run the scenario method
* run the check method
"""
# use mock bitcoind
virtualchain.setup_virtualchain( blockstack_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind )
# set up blockstack
# NOTE: utxo_opts encodes the mock-bitcoind options
blockstack_opts, bitcoin_opts, utxo_opts, dht_opts = blockstack.lib.configure( config_file=config_file, interactive=False )
# override multiprocessing options to ensure single-process behavior
utxo_opts['multiprocessing_num_procs'] = 1
utxo_opts['multiprocessing_num_blocks'] = 64
blockstackd.set_bitcoin_opts( bitcoin_opts )
blockstackd.set_utxo_opts( utxo_opts )
db = blockstackd.get_state_engine()
bitcoind = mock_bitcoind.connect_mock_bitcoind( utxo_opts )
sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain( utxo_opts, bitcoind.getblockcount(), db )
mock_utxo = blockstack.lib.connect_utxo_provider( utxo_opts )
working_dir = virtualchain.get_working_dir()
# set up test environment
testlib.set_utxo_client( mock_utxo )
testlib.set_utxo_opts( utxo_opts )
testlib.set_bitcoind( bitcoind )
testlib.set_state_engine( db )
test_env = {
"sync_virtualchain_upcall": sync_virtualchain_upcall,
"working_dir": working_dir
}
# sync initial utxos
testlib.next_block( **test_env )
# load the scenario into the mock blockchain and mock utxo provider
try:
scenario.scenario( scenario.wallets, **test_env )
except Exception, e:
log.exception(e)
traceback.print_exc()
log.error("Failed to run scenario '%s'" % scenario.__name__)
return False
def sync_blockchain( bt_opts, last_block, expected_snapshots={}, **virtualchain_args ):
"""
synchronize state with the blockchain.
Return True on success
Return False if we're supposed to stop indexing
Abort on error
"""
# make this usable even if we haven't explicitly configured virtualchain
impl = sys.modules[__name__]
if virtualchain.get_implementation() is not None:
impl = None
log.info("Synchronizing database up to block %s" % last_block)
db_filename = virtualchain.get_db_filename(impl=impl)
# NOTE: this is the only place where a read-write handle should be created,
# since this is the only place where the db should be modified.
new_db = BlockstackDB.borrow_readwrite_instance( db_filename, last_block, expected_snapshots=expected_snapshots )
rc = virtualchain.sync_virtualchain( bt_opts, last_block, new_db, expected_snapshots=expected_snapshots, **virtualchain_args )
BlockstackDB.release_readwrite_instance( new_db, last_block )
return rc
"bitcoind_user": "******",
"bitcoind_passwd": "talos",
"bitcoind_server": "127.0.0.1",
"bitcoind_p2p_port": 18444,
"bitcoind_spv_path": "./tmp.dat"
}
"""
conf={"bitcoind_port":13001,
"bitcoind_user":"******",
"bitcoind_passwd":"13000",
"bitcoind_server":"13.93.113.195",
"bitcoind_p2p_port":13000,
"bitcoind_spv_path":"./tmp.dat"}
"""
"""
conf={"bitcoind_port":8332,
"bitcoind_user":"******",
"bitcoind_passwd":"7876aeeebedebbc156b7dd69f9865af0fd8db6bc2afa6dc9eb64061ce9af91cc",
"bitcoind_server":"127.0.0.1",
"bitcoind_p2p_port":8333,
"bitcoind_spv_path":"./tmp.dat"}
"""
# virtualchain.connect_bitcoind(conf)
engine = StateEngine(util.MAGIC_BYTES,
util.OPCODES,
util.OPCODE_FIELDS,
impl=chain)
virtualchain.sync_virtualchain(conf, 879, engine)
def run_zonefilemanage():
working_dir = "/tmp/zonefilemanage"
# errase prior state
if os.path.exists(working_dir):
log.debug("Remove %s " % working_dir)
shutil.rmtree(working_dir)
if not os.path.exists(working_dir):
os.makedirs(working_dir)
# export to test
os.environ["VIRTUALCHAIN_WORKING_DIR"] = working_dir
# set up bitcoind
bitcoind_regtest_reset()
virtualchain_working_dir = os.environ["VIRTUALCHAIN_WORKING_DIR"]
spv_header_path = os.path.join(virtualchain_working_dir, "spv_headers.dat")
virtualchain.setup_virtualchain(state_engine)
# db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW)
log.info("Connect to the bitcoind ")
# Start the pinger
pinger = Pinger()
pinger.start()
bitcoind = bitcoin_regtest_connect(bitcoin_regtest_opts())
if is_main_worker():
log.info("fill up the default wallet")
# set up the default payment wallet
default_payment_wallet = MultisigWallet(2, '5JYAj69z2GuFAZHrkhRuBKoCmKh6GcPXgcw9pbH8e8J2pu2RU9z',
'5Kfg4xkZ1gGN5ozgDZ37Mn3EH9pXSuWZnQt1pzax4cLax8PetNs',
'5JXB7rNxZa8yQtpuKtwy1nWUUTgdDEYTDmaEqQvKKC8HCWs64bL')
# load wallets
bitcoion_regtest_fill_wallets(wallets, default_payment_wallet=default_payment_wallet)
else:
# Watch out for wallets
for wallet in wallets:
testnet_wif = wallet.privkey
if not testnet_wif.startswith("c"):
testnet_wif = virtualchain.BitcoinPrivateKey(testnet_wif).to_wif()
bitcoind.importprivkey(testnet_wif, "")
addr = virtualchain.BitcoinPublicKey(wallet.pubkey_hex).address()
log.info("Watch out for %s" % (addr))
for wallet in wallets:
addr = get_wallet_addr(wallet)
unspents = bitcoind.listunspent(0, 200000, [addr])
SATOSHIS_PER_COIN = 10 ** 8
value = sum([int(round(s["amount"] * SATOSHIS_PER_COIN)) for s in unspents])
print >> sys.stderr, "Address %s loaded with %s satoshis" % (addr, value)
db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW)
# Kill the pid for use this port
return_code, output = commands.getstatusoutput("netstat -apn | grep %s | grep python| awk '{print $7}'" % RPC_SERVER_PORT)
if 'python' in output:
import re
pattern = re.compile("(\d+)/python")
match = pattern.search(output)
if match:
port = match.group(1)
rc = os.system("kill -9 %s" % port)
if rc != 0:
log.exception("force kill failed")
os.abort()
# Start up the rpc server
server = VoteServer()
server.start()
set_global_server(server)
while True:
height = bitcoind.getblockcount()
log.info("Sync virtualchain up to %s " % height)
virtualchain.sync_virtualchain(bitcoin_regtest_opts(), height, db)
# wait for the next block
deadline = time.time() + REINDEX_FREQUENCY
while time.time() < deadline:
try:
time.sleep(1)
except:
break
def run_scenario(scenario, config_file):
"""
Run a test scenario:
* set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain
* seed it with the initial values in the wallet
* set the initial consensus hash
* run the scenario method
* run the check method
"""
mock_bitcoind_save_path = "/tmp/mock_bitcoind.dat"
if os.path.exists(mock_bitcoind_save_path):
try:
os.unlink(mock_bitcoind_save_path)
except:
pass
# use mock bitcoind
worker_env = {
# use mock_bitcoind to connect to bitcoind (but it has to import it in order to use it)
"VIRTUALCHAIN_MOD_CONNECT_BLOCKCHAIN": mock_bitcoind.__file__,
"MOCK_BITCOIND_SAVE_PATH": mock_bitcoind_save_path,
"BLOCKSTORE_TEST": "1"
}
if os.environ.get("PYTHONPATH", None) is not None:
worker_env["PYTHONPATH"] = os.environ["PYTHONPATH"]
virtualchain.setup_virtualchain(
blockstore_state_engine,
bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind,
index_worker_env=worker_env)
# set up blockstore
# NOTE: utxo_opts encodes the mock-bitcoind options
blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure(
config_file=config_file, interactive=False)
# override multiprocessing options to ensure single-process behavior
utxo_opts['multiprocessing_num_procs'] = 1
utxo_opts['multiprocessing_num_blocks'] = 10
# pass along extra arguments
utxo_opts['save_file'] = mock_bitcoind_save_path
# save headers as well
utxo_opts['spv_headers_path'] = mock_bitcoind_save_path + ".spvheaders"
with open(utxo_opts['spv_headers_path'], "w") as f:
# write out "initial" headers, up to the first block
empty_header = ("00" * 81).decode('hex')
for i in xrange(0, blockstore.FIRST_BLOCK_MAINNET):
f.write(empty_header)
blockstored.set_bitcoin_opts(bitcoin_opts)
blockstored.set_utxo_opts(utxo_opts)
db = blockstored.get_state_engine()
bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts)
sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain(
utxo_opts, bitcoind.getblockcount(), db)
mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts)
working_dir = virtualchain.get_working_dir()
# set up test environment
testlib.set_utxo_opts(utxo_opts)
testlib.set_utxo_client(mock_utxo)
testlib.set_bitcoind(bitcoind)
testlib.set_state_engine(db)
test_env = {
"sync_virtualchain_upcall": sync_virtualchain_upcall,
"working_dir": working_dir,
"bitcoind": bitcoind,
"bitcoind_save_path": mock_bitcoind_save_path
}
# sync initial utxos
testlib.next_block(**test_env)
try:
os.unlink(mock_bitcoind_save_path)
except:
pass
# load the scenario into the mock blockchain and mock utxo provider
try:
scenario.scenario(scenario.wallets, **test_env)
except Exception, e:
log.exception(e)
traceback.print_exc()
log.error("Failed to run scenario '%s'" % scenario.__name__)
return False
# count blocks as given (needs to be by ref)
block_counter = {'count': virtualchain.get_first_block_id() }
# load the scenario into the mock blockchain and mock utxo provider
try:
scenario.scenario( scenario.wallets, bitcoind=bitcoind, block_counter=block_counter )
except Exception, e:
log.exception(e)
log.error("Failed to run scenario '%s'" % scenario.__name__)
return False
# one more, just to cap it off
testlib.next_block( bitcoind=bitcoind, block_counter=block_counter )
# crawl the mock blockchain
virtualchain.sync_virtualchain( utxo_opts, block_counter['count'], db )
# run the checks on the database
try:
rc = scenario.check( db )
return rc
except Exception, e:
log.exception(e)
log.error("Failed to run tests '%s'" % scenario.__name__)
return False
return rc
if __name__ == "__main__":
def run_server( foreground=False):
"""
Run the blockstored RPC server, optionally in the foreground.
"""
signal.signal( signal.SIGINT, sigint_handler_server )
bitcoin_opts = get_bitcoin_opts()
tac_file = get_tacfile_path()
log_file = get_logfile_path()
pid_file = get_pidfile_path()
start_block, current_block = get_index_range()
indexer_command = "%s indexer" % sys.argv[0]
if foreground:
command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
else:
command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file,
log_file,
tac_file)
if start_block != current_block:
# bring us up to speed
log.info("Synchronizing with blockchain, up to %s" % current_block )
blockstore_state_engine = get_state_engine()
virtualchain.sync_virtualchain( bitcoin_opts, current_block, blockstore_state_engine )
try:
# fork the server
blockstored = subprocess.Popen( command, shell=True, preexec_fn=os.setsid)
# fork the indexer
indexer = subprocess.Popen( indexer_command, shell=True )
log.info('Blockstored successfully started')
# wait for it to die
blockstored.wait()
# stop our indexing thread
os.kill( indexer.pid, signal.SIGINT )
indexer.wait()
return blockstored.returncode
except IndexError, ie:
traceback.print_exc()
# indicates that we don't have the latest block
log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \
"Please verify that your bitcoin provider has processd up to" + \
"to block %s.\n" % (START_BLOCK) + \
" Example: bitcoin-cli getblockcount" )
try:
os.killpg(blockstored.pid, signal.SIGTERM)
except:
pass
exit(1)
def run_scenario(scenario, config_file):
"""
Run a test scenario:
* set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain
* seed it with the initial values in the wallet
* set the initial consensus hash
* run the scenario method
* run the check method
"""
mock_bitcoind_save_path = "/tmp/mock_bitcoind.dat"
if os.path.exists(mock_bitcoind_save_path):
try:
os.unlink(mock_bitcoind_save_path)
except:
pass
# use mock bitcoind
worker_env = {
# use mock_bitcoind to connect to bitcoind (but it has to import it in order to use it)
"VIRTUALCHAIN_MOD_CONNECT_BLOCKCHAIN": mock_bitcoind.__file__,
"MOCK_BITCOIND_SAVE_PATH": mock_bitcoind_save_path,
"BLOCKSTORE_TEST": "1",
}
if os.environ.get("PYTHONPATH", None) is not None:
worker_env["PYTHONPATH"] = os.environ["PYTHONPATH"]
virtualchain.setup_virtualchain(
blockstore_state_engine,
bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind,
index_worker_env=worker_env,
)
# set up blockstore
# NOTE: utxo_opts encodes the mock-bitcoind options
blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure(
config_file=config_file, interactive=False
)
# override multiprocessing options to ensure single-process behavior
utxo_opts["multiprocessing_num_procs"] = 1
utxo_opts["multiprocessing_num_blocks"] = 10
# pass along extra arguments
utxo_opts["save_file"] = mock_bitcoind_save_path
# save headers as well
utxo_opts["spv_headers_path"] = mock_bitcoind_save_path + ".spvheaders"
with open(utxo_opts["spv_headers_path"], "w") as f:
# write out "initial" headers, up to the first block
empty_header = ("00" * 81).decode("hex")
for i in xrange(0, blockstore.FIRST_BLOCK_MAINNET):
f.write(empty_header)
blockstored.set_bitcoin_opts(bitcoin_opts)
blockstored.set_utxo_opts(utxo_opts)
db = blockstored.get_state_engine()
bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts)
sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain(utxo_opts, bitcoind.getblockcount(), db)
mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts)
working_dir = virtualchain.get_working_dir()
# set up test environment
testlib.set_utxo_opts(utxo_opts)
testlib.set_utxo_client(mock_utxo)
testlib.set_bitcoind(bitcoind)
testlib.set_state_engine(db)
test_env = {
"sync_virtualchain_upcall": sync_virtualchain_upcall,
"working_dir": working_dir,
"bitcoind": bitcoind,
"bitcoind_save_path": mock_bitcoind_save_path,
}
# sync initial utxos
testlib.next_block(**test_env)
try:
os.unlink(mock_bitcoind_save_path)
except:
pass
# load the scenario into the mock blockchain and mock utxo provider
try:
scenario.scenario(scenario.wallets, **test_env)
except Exception, e:
log.exception(e)
traceback.print_exc()
log.error("Failed to run scenario '%s'" % scenario.__name__)
return False
# foreground
api_server_command = ('twistd --pidfile=%s -noy %s' % (pid_file, tac_file)).split()
# start API server
blockstored = subprocess.Popen( api_server_command, shell=False)
set_indexing( False )
if start_block != current_block:
# bring us up to speed
set_indexing( True )
blockstore_state_engine = get_state_engine()
virtualchain.sync_virtualchain( bt_opts, current_block, blockstore_state_engine )
set_indexing( False )
# fork the indexer
if foreground:
indexer = subprocess.Popen( indexer_command, shell=False )
else:
indexer = subprocess.Popen( indexer_command, shell=False, stdout=logfile, stderr=logfile )
indexer_pid = indexer.pid
# wait for the API server to die (we kill it with `blockstored stop`)
blockstored.wait()
# stop our indexer subprocess
