def get_logfile_path(): """ Get the logfile path for our service endpoint. """ working_dir = virtualchain.get_working_dir() logfile_filename = blockstore_state_engine.get_virtual_chain_name() + ".log" return os.path.join( working_dir, logfile_filename )
def get_pidfile_path(): """ Get the PID file path. """ working_dir = virtualchain.get_working_dir() pid_filename = blockstore_state_engine.get_virtual_chain_name() + ".pid" return os.path.join( working_dir, pid_filename )
def restore( snapshot_path, restore_dir, pubkeys, num_required ):
global value_hashes
config_path = os.environ.get("BLOCKSTACK_CLIENT_CONFIG")
assert config_path
os.makedirs(restore_dir)
shutil.copy(config_path, os.path.join(restore_dir, os.path.basename(config_path)))
rc = ysi.fast_sync_import( restore_dir, "file://{}".format(snapshot_path), public_keys=pubkeys, num_required=num_required )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
return False
# database must be identical
db_filenames = ['ysi-server.db', 'ysi-server.snapshots', 'ysi-server.lastblock']
src_paths = [os.path.join(virtualchain.get_working_dir(), fn) for fn in db_filenames]
backup_paths = [os.path.join(restore_dir, fn) for fn in db_filenames]
for src_path, backup_path in zip(src_paths, backup_paths):
rc = os.system("cmp '{}' '{}'".format(src_path, backup_path))
if rc != 0:
print '{} disagress with {}'.format(src_path, backup_path)
return False
# all zone files must be present
for vh in value_hashes:
zfdata = ysi.get_cached_zonefile_data(vh, zonefile_dir=os.path.join(restore_dir, 'zonefiles'))
if zfdata is None:
print 'Missing {} in {}'.format(vh, os.path.join(restore_dir, 'zonefiles'))
return False
shutil.rmtree(restore_dir)
return True
def restore( snapshot_path, restore_dir, pubkeys, num_required ):
global value_hashes
config_path = os.environ.get("BLOCKSTACK_CLIENT_CONFIG")
assert config_path
os.makedirs(restore_dir)
shutil.copy(config_path, os.path.join(restore_dir, os.path.basename(config_path)))
rc = blockstack.fast_sync_import( restore_dir, "file://{}".format(snapshot_path), public_keys=pubkeys, num_required=num_required )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
return False
# database must be identical
db_filenames = ['blockstack-server.db', 'blockstack-server.snapshots', 'blockstack-server.lastblock']
src_paths = [os.path.join(virtualchain.get_working_dir(), fn) for fn in db_filenames]
backup_paths = [os.path.join(restore_dir, fn) for fn in db_filenames]
for src_path, backup_path in zip(src_paths, backup_paths):
rc = os.system("cmp '{}' '{}'".format(src_path, backup_path))
if rc != 0:
print '{} disagress with {}'.format(src_path, backup_path)
return False
# all zone files must be present
for vh in value_hashes:
zfdata = blockstack.get_cached_zonefile_data(vh, zonefile_dir=os.path.join(restore_dir, 'zonefiles'))
if zfdata is None:
print 'Missing {} in {}'.format(vh, os.path.join(restore_dir, 'zonefiles'))
return False
shutil.rmtree(restore_dir)
return True
def run_scenario(scenario, config_file):
"""
Run a test scenario:
* set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain
* seed it with the initial values in the wallet
* set the initial consensus hash
* run the scenario method
* run the check method
"""
# use mock bitcoind
virtualchain.setup_virtualchain(
blockstore_state_engine,
bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind)
# set up blockstore
# NOTE: utxo_opts encodes the mock-bitcoind options
blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure(
config_file=config_file, interactive=False)
# override multiprocessing options to ensure single-process behavior
utxo_opts['multiprocessing_num_procs'] = 1
utxo_opts['multiprocessing_num_blocks'] = 64
blockstored.set_bitcoin_opts(bitcoin_opts)
blockstored.set_utxo_opts(utxo_opts)
db = blockstored.get_state_engine()
bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts)
sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain(
utxo_opts, bitcoind.getblockcount(), db)
mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts)
working_dir = virtualchain.get_working_dir()
# set up test environment
testlib.set_utxo_client(mock_utxo)
testlib.set_bitcoind(bitcoind)
testlib.set_state_engine(db)
test_env = {
"sync_virtualchain_upcall": sync_virtualchain_upcall,
"working_dir": working_dir
}
# sync initial utxos
testlib.next_block(**test_env)
# load the scenario into the mock blockchain and mock utxo provider
try:
scenario.scenario(scenario.wallets, **test_env)
except Exception, e:
log.exception(e)
traceback.print_exc()
log.error("Failed to run scenario '%s'" % scenario.__name__)
return False
def get_zonefile_dir( working_dir=None ):
"""
Get the path to the directory to hold any zonefiles we download.
"""
if working_dir is None:
working_dir = virtualchain.get_working_dir()
zonefile_dir = os.path.join( working_dir, "zonefiles" )
return zonefile_dir
def get_announce_filename( working_dir=None ):
"""
Get the path to the file that stores all of the announcements.
"""
if working_dir is None:
working_dir = virtualchain.get_working_dir()
announce_filepath = os.path.join( working_dir, virtualchain.get_implementation().get_virtual_chain_name() ) + ".announce"
return announce_filepath
def get_announce_filename( working_dir=None ):
"""
Get the path to the file that stores all of the announcements.
"""
if working_dir is None:
working_dir = virtualchain.get_working_dir()
announce_filepath = os.path.join( working_dir, virtualchain.get_implementation().get_virtual_chain_name() ) + ".announce"
return announce_filepath
def get_zonefile_dir( working_dir=None ):
"""
Get the path to the directory to hold any zonefiles we download.
"""
if working_dir is None:
working_dir = virtualchain.get_working_dir()
zonefile_dir = os.path.join( working_dir, "zonefiles" )
return zonefile_dir
def get_testset_filename( working_dir=None ):
"""
Get the path to the file to determine whether or not we're in testset.
"""
if working_dir is None:
working_dir = virtualchain.get_working_dir()
testset_filepath = os.path.join( working_dir, virtualchain.get_implementation().get_virtual_chain_name() ) + ".testset"
return testset_filepath
def run_scenario( scenario, config_file ):
"""
Run a test scenario:
* set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain
* seed it with the initial values in the wallet
* set the initial consensus hash
* run the scenario method
* run the check method
"""
# use mock bitcoind
virtualchain.setup_virtualchain( blockstack_state_engine, bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind )
# set up blockstack
# NOTE: utxo_opts encodes the mock-bitcoind options
blockstack_opts, bitcoin_opts, utxo_opts, dht_opts = blockstack.lib.configure( config_file=config_file, interactive=False )
# override multiprocessing options to ensure single-process behavior
utxo_opts['multiprocessing_num_procs'] = 1
utxo_opts['multiprocessing_num_blocks'] = 64
blockstackd.set_bitcoin_opts( bitcoin_opts )
blockstackd.set_utxo_opts( utxo_opts )
db = blockstackd.get_state_engine()
bitcoind = mock_bitcoind.connect_mock_bitcoind( utxo_opts )
sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain( utxo_opts, bitcoind.getblockcount(), db )
mock_utxo = blockstack.lib.connect_utxo_provider( utxo_opts )
working_dir = virtualchain.get_working_dir()
# set up test environment
testlib.set_utxo_client( mock_utxo )
testlib.set_utxo_opts( utxo_opts )
testlib.set_bitcoind( bitcoind )
testlib.set_state_engine( db )
test_env = {
"sync_virtualchain_upcall": sync_virtualchain_upcall,
"working_dir": working_dir
}
# sync initial utxos
testlib.next_block( **test_env )
# load the scenario into the mock blockchain and mock utxo provider
try:
scenario.scenario( scenario.wallets, **test_env )
except Exception, e:
log.exception(e)
traceback.print_exc()
log.error("Failed to run scenario '%s'" % scenario.__name__)
return False
def get_default_talos_config():
import talosvirtualchain
return {
"bitcoind_port":
18332,
"bitcoind_user":
"******",
"bitcoind_passwd":
"talos",
"bitcoind_server":
"127.0.0.1",
"bitcoind_p2p_port":
18444,
"bitcoind_spv_path":
os.path.join(get_working_dir(impl=talosvirtualchain), "tmp.dat")
}
def get_indexing_lockfile():
"""
Return path to the indexing lockfile
"""
return os.path.join( virtualchain.get_working_dir(), "blockstore.indexing" )
def get_indexing_lockfile(impl=None):
"""
Return path to the indexing lockfile
"""
return os.path.join( virtualchain.get_working_dir(impl=impl), "blockstack-server.indexing" )
def store_announcement( announcement_hash, announcement_text, working_dir=None, force=False ):
"""
Store a new announcement locally, atomically.
"""
if working_dir is None:
working_dir = virtualchain.get_working_dir()
if not force:
# don't store unless we haven't seen it before
if announcement_hash in ANNOUNCEMENTS:
return
announce_filename = get_announce_filename( working_dir )
announce_filename_tmp = announce_filename + ".tmp"
announce_text = ""
announce_cleanup_list = []
# did we try (and fail) to store a previous announcement? If so, merge them all
if os.path.exists( announce_filename_tmp ):
log.debug("Merge announcement list %s" % announce_filename_tmp )
with open(announce_filename, "r") as f:
announce_text += f.read()
i = 1
failed_path = announce_filename_tmp + (".%s" % i)
while os.path.exists( failed_path ):
log.debug("Merge announcement list %s" % failed_paht )
with open(failed_path, "r") as f:
announce_text += f.read()
announce_cleanup_list.append( failed_path )
i += 1
failed_path = announce_filename_tmp + (".%s" % i)
announce_filename_tmp = failed_path
if os.path.exists( announce_filename ):
with open(announce_filename, "r" ) as f:
announce_text += f.read()
announce_text += ("\n%s\n" % announcement_hash)
# filter
if not force:
announcement_list = announce_text.split("\n")
unseen_announcements = filter( lambda a: a not in ANNOUNCEMENTS, announcement_list )
announce_text = "\n".join( unseen_announcements ).strip() + "\n"
log.debug("Store announcement hash to %s" % announce_filename )
with open(announce_filename_tmp, "w" ) as f:
f.write( announce_text )
f.flush()
# NOTE: rename doesn't remove the old file on Windows
if sys.platform == 'win32' and os.path.exists( announcement_filename_tmp ):
try:
os.unlink( announcement_filename_tmp )
except:
pass
try:
os.rename( announce_filename_tmp, announce_filename )
except:
log.error("Failed to save announcement %s to %s" % (announcement_hash, announce_filename ))
raise
# clean up
for tmp_path in announce_cleanup_list:
try:
os.unlink( tmp_path )
except:
pass
# put the announcement text
announcement_text_dir = os.path.join( working_dir, "announcements" )
if not os.path.exists( announcement_text_dir ):
try:
os.makedirs( announcement_text_dir )
except:
log.error("Failed to make directory %s" % announcement_text_dir )
raise
announcement_text_path = os.path.join( announcement_text_dir, "%s.txt" % announcement_hash )
try:
with open( announcement_text_path, "w" ) as f:
f.write( announcement_text )
except:
log.error("Failed to save announcement text to %s" % announcement_text_path )
raise
log.debug("Stored announcement to %s" % (announcement_text_path))
def run_scenario(scenario, config_file):
"""
Run a test scenario:
* set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain
* seed it with the initial values in the wallet
* set the initial consensus hash
* run the scenario method
* run the check method
"""
mock_bitcoind_save_path = "/tmp/mock_bitcoind.dat"
if os.path.exists(mock_bitcoind_save_path):
try:
os.unlink(mock_bitcoind_save_path)
except:
pass
# use mock bitcoind
worker_env = {
# use mock_bitcoind to connect to bitcoind (but it has to import it in order to use it)
"VIRTUALCHAIN_MOD_CONNECT_BLOCKCHAIN": mock_bitcoind.__file__,
"MOCK_BITCOIND_SAVE_PATH": mock_bitcoind_save_path,
"BLOCKSTORE_TEST": "1"
}
if os.environ.get("PYTHONPATH", None) is not None:
worker_env["PYTHONPATH"] = os.environ["PYTHONPATH"]
virtualchain.setup_virtualchain(
blockstore_state_engine,
bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind,
index_worker_env=worker_env)
# set up blockstore
# NOTE: utxo_opts encodes the mock-bitcoind options
blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure(
config_file=config_file, interactive=False)
# override multiprocessing options to ensure single-process behavior
utxo_opts['multiprocessing_num_procs'] = 1
utxo_opts['multiprocessing_num_blocks'] = 10
# pass along extra arguments
utxo_opts['save_file'] = mock_bitcoind_save_path
# save headers as well
utxo_opts['spv_headers_path'] = mock_bitcoind_save_path + ".spvheaders"
with open(utxo_opts['spv_headers_path'], "w") as f:
# write out "initial" headers, up to the first block
empty_header = ("00" * 81).decode('hex')
for i in xrange(0, blockstore.FIRST_BLOCK_MAINNET):
f.write(empty_header)
blockstored.set_bitcoin_opts(bitcoin_opts)
blockstored.set_utxo_opts(utxo_opts)
db = blockstored.get_state_engine()
bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts)
sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain(
utxo_opts, bitcoind.getblockcount(), db)
mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts)
working_dir = virtualchain.get_working_dir()
# set up test environment
testlib.set_utxo_opts(utxo_opts)
testlib.set_utxo_client(mock_utxo)
testlib.set_bitcoind(bitcoind)
testlib.set_state_engine(db)
test_env = {
"sync_virtualchain_upcall": sync_virtualchain_upcall,
"working_dir": working_dir,
"bitcoind": bitcoind,
"bitcoind_save_path": mock_bitcoind_save_path
}
# sync initial utxos
testlib.next_block(**test_env)
try:
os.unlink(mock_bitcoind_save_path)
except:
pass
# load the scenario into the mock blockchain and mock utxo provider
try:
scenario.scenario(scenario.wallets, **test_env)
except Exception, e:
log.exception(e)
traceback.print_exc()
log.error("Failed to run scenario '%s'" % scenario.__name__)
return False
def default_blockstack_opts( config_file=None, testset=False ):
"""
Get our default blockstack opts from a config file
or from sane defaults.
"""
if config_file is None:
config_file = virtualchain.get_config_filename()
testset_path = get_testset_filename( virtualchain.get_working_dir() )
announce_path = get_announce_filename( virtualchain.get_working_dir() )
parser = SafeConfigParser()
parser.read( config_file )
blockstack_opts = {}
tx_broadcaster = None
utxo_provider = None
testset_first_block = None
max_subsidy = 0
contact_email = None
announcers = "judecn.id,muneeb.id,shea256.id"
announcements = None
if parser.has_section('blockstack'):
if parser.has_option('blockstack', 'tx_broadcaster'):
tx_broadcaster = parser.get('blockstack', 'tx_broadcaster')
if parser.has_option('blockstack', 'utxo_provider'):
utxo_provider = parser.get('blockstack', 'utxo_provider')
if parser.has_option('blockstack', 'testset_first_block'):
testset_first_block = int( parser.get('blockstack', 'testset_first_block') )
if parser.has_option('blockstack', 'max_subsidy'):
max_subsidy = int( parser.get('blockstack', 'max_subsidy'))
if parser.has_option('blockstack', 'email'):
contact_email = parser.get('blockstack', 'email')
if parser.has_option('blockstack', 'announcers'):
# must be a CSV of blockchain IDs
announcer_list_str = parser.get('blockstack', 'announcers')
announcer_list = announcer_list_str.split(",")
import scripts
# validate each one
valid = True
for bid in announcer_list:
if not scripts.is_name_valid( bid ):
log.error("Invalid blockchain ID '%s'" % bid)
valid = False
if valid:
announcers = ",".join(announcer_list)
if os.path.exists( testset_path ):
# testset file flag set
testset = True
if os.path.exists( announce_path ):
# load announcement list
with open( announce_path, "r" ) as f:
announce_text = f.readlines()
all_announcements = [ a.strip() for a in announce_text ]
unseen_announcements = []
# find announcements we haven't seen yet
for a in all_announcements:
if a not in ANNOUNCEMENTS:
unseen_announcements.append( a )
announcements = ",".join( unseen_announcements )
blockstack_opts = {
'tx_broadcaster': tx_broadcaster,
'utxo_provider': utxo_provider,
'testset': testset,
'testset_first_block': testset_first_block,
'max_subsidy': max_subsidy,
'email': contact_email,
'announcers': announcers,
'announcements': announcements
}
# strip Nones
for (k, v) in blockstack_opts.items():
if v is None:
del blockstack_opts[k]
return blockstack_opts
def fast_sync_import( working_dir, import_url, public_keys=config.FAST_SYNC_PUBLIC_KEYS, num_required=len(config.FAST_SYNC_PUBLIC_KEYS), verbose=False ):
"""
Fast sync import.
Verify the given fast-sync file from @import_path using @public_key, and then
uncompress it into @working_dir.
Verify that at least `num_required` public keys in `public_keys` signed.
NOTE: `public_keys` needs to be in the same order as the private keys that signed.
"""
def logmsg(s):
if verbose:
print s
else:
log.debug(s)
def logerr(s):
if verbose:
print >> sys.stderr, s
else:
log.error(s)
if working_dir is None:
working_dir = virtualchain.get_working_dir()
if not os.path.exists(working_dir):
logerr("No such directory {}".format(working_dir))
return False
# go get it
import_path = fast_sync_fetch(import_url)
if import_path is None:
logerr("Failed to fetch {}".format(import_url))
return False
# format: <signed bz2 payload> <sigb64> <sigb64 length (8 bytes hex)> ... <num signatures>
file_size = 0
try:
sb = os.stat(import_path)
file_size = sb.st_size
except Exception as e:
log.exception(e)
return False
num_signatures = 0
ptr = file_size
signatures = []
with open(import_path, 'r') as f:
info = fast_sync_inspect( f )
if 'error' in info:
logerr("Failed to inspect snapshot {}: {}".format(import_path, info['error']))
return False
signatures = info['signatures']
ptr = info['payload_size']
# get the hash of the file
hash_hex = get_file_hash(f, hashlib.sha256, fd_len=ptr)
# validate signatures over the hash
logmsg("Verify {} bytes".format(ptr))
key_idx = 0
num_match = 0
for next_pubkey in public_keys:
for sigb64 in signatures:
valid = verify_digest( hash_hex, keylib.ECPublicKey(next_pubkey).to_hex(), sigb64, hashfunc=hashlib.sha256 )
if valid:
num_match += 1
if num_match >= num_required:
break
logmsg("Public key {} matches {} ({})".format(next_pubkey, sigb64, hash_hex))
signatures.remove(sigb64)
else:
logmsg("Public key {} does NOT match {} ({})".format(next_pubkey, sigb64, hash_hex))
# enough signatures?
if num_match < num_required:
logerr("Not enough signatures match (required {}, found {})".format(num_required, num_match))
return False
# decompress
import_path = os.path.abspath(import_path)
res = fast_sync_snapshot_decompress(import_path, working_dir)
if 'error' in res:
logerr("Failed to decompress {} to {}: {}".format(import_path, working_dir, res['error']))
return False
# restore from backup
rc = ysi_backup_restore(working_dir, None)
if not rc:
logerr("Failed to instantiate ysi name database")
return False
# success!
logmsg("Restored to {}".format(working_dir))
return True
def default_blockstack_opts( config_file=None, virtualchain_impl=None ):
"""
Get our default blockstack opts from a config file
or from sane defaults.
"""
if config_file is None:
config_file = virtualchain.get_config_filename()
announce_path = get_announce_filename( virtualchain.get_working_dir(impl=virtualchain_impl) )
parser = SafeConfigParser()
parser.read( config_file )
blockstack_opts = {}
contact_email = None
announcers = "judecn.id,muneeb.id,shea256.id"
announcements = None
backup_frequency = 144 # once a day; 10 minute block time
backup_max_age = 1008 # one week
rpc_port = RPC_SERVER_PORT
serve_zonefiles = True
serve_profiles = False
serve_data = False
zonefile_dir = os.path.join( os.path.dirname(config_file), "zonefiles")
analytics_key = None
zonefile_storage_drivers = "disk,dht"
zonefile_storage_drivers_write = "disk"
profile_storage_drivers = "disk"
profile_storage_drivers_write = "disk"
data_storage_drivers = "disk"
data_storage_drivers_write = "disk"
redirect_data = False
data_servers = None
server_version = None
atlas_enabled = True
atlas_seed_peers = "node.blockstack.org:%s" % RPC_SERVER_PORT
atlasdb_path = os.path.join( os.path.dirname(config_file), "atlas.db" )
atlas_blacklist = ""
atlas_hostname = socket.gethostname()
if parser.has_section('blockstack'):
if parser.has_option('blockstack', 'backup_frequency'):
backup_frequency = int( parser.get('blockstack', 'backup_frequency'))
if parser.has_option('blockstack', 'backup_max_age'):
backup_max_age = int( parser.get('blockstack', 'backup_max_age') )
if parser.has_option('blockstack', 'email'):
contact_email = parser.get('blockstack', 'email')
if parser.has_option('blockstack', 'rpc_port'):
rpc_port = int(parser.get('blockstack', 'rpc_port'))
if parser.has_option('blockstack', 'serve_zonefiles'):
serve_zonefiles = parser.get('blockstack', 'serve_zonefiles')
if serve_zonefiles.lower() in ['1', 'yes', 'true', 'on']:
serve_zonefiles = True
else:
serve_zonefiles = False
if parser.has_option('blockstack', 'serve_profiles'):
serve_profiles = parser.get('blockstack', 'serve_profiles')
if serve_profiles.lower() in ['1', 'yes', 'true', 'on']:
serve_profiles = True
else:
serve_profiles = False
if parser.has_option('blockstack', 'serve_data'):
serve_data = parser.get('blockstack', 'serve_data')
if serve_data.lower() in ['1', 'yes', 'true', 'on']:
serve_data = True
else:
serve_data = False
if parser.has_option("blockstack", "zonefile_storage_drivers"):
zonefile_storage_drivers = parser.get("blockstack", "zonefile_storage_drivers")
if parser.has_option("blockstack", "zonefile_storage_drivers_write"):
zonefile_storage_drivers_write = parser.get("blockstack", "zonefile_storage_drivers_write")
if parser.has_option("blockstack", "profile_storage_drivers"):
profile_storage_drivers = parser.get("blockstack", "profile_storage_drivers")
if parser.has_option("blockstack", "profile_storage_drivers_write"):
profile_storage_drivers_write = parser.get("blockstack", "profile_storage_drivers_write")
if parser.has_option("blockstack", "data_storage_drivers"):
data_storage_drivers = parser.get("blockstack", "data_storage_drivers")
if parser.has_option("blockstack", "data_storage_drivers_write"):
data_storage_drivers_write = parser.get("blockstack", "data_storage_drivers_write")
if parser.has_option("blockstack", "zonefiles"):
zonefile_dir = parser.get("blockstack", "zonefiles")
if parser.has_option('blockstack', 'redirect_data'):
redirect_data = parser.get('blockstack', 'redirect_data')
if redirect_data.lower() in ['1', 'yes', 'true', 'on']:
redirect_data = True
else:
redirect_data = False
if parser.has_option('blockstack', 'data_servers'):
data_servers = parser.get('blockstack', 'data_servers')
# must be a CSV of host:port
hostports = filter( lambda x: len(x) > 0, data_servers.split(",") )
for hp in hostports:
host, port = url_to_host_port( hp )
assert host is not None and port is not None
if parser.has_option('blockstack', 'announcers'):
# must be a CSV of blockchain IDs
announcer_list_str = parser.get('blockstack', 'announcers')
announcer_list = filter( lambda x: len(x) > 0, announcer_list_str.split(",") )
import scripts
# validate each one
valid = True
for bid in announcer_list:
if not scripts.is_name_valid( bid ):
log.error("Invalid blockchain ID '%s'" % bid)
valid = False
if valid:
announcers = ",".join(announcer_list)
if parser.has_option('blockstack', 'analytics_key'):
analytics_key = parser.get('blockstack', 'analytics_key')
if parser.has_option('blockstack', 'server_version'):
server_version = parser.get('blockstack', 'server_version')
if parser.has_option('blockstack', 'atlas'):
atlas_enabled = parser.get('blockstack', 'atlas')
if atlas_enabled.lower() in ['true', '1', 'enabled', 'enabled', 'on']:
atlas_enabled = True
else:
atlas_enabled = False
if parser.has_option('blockstack', 'atlas_seeds'):
atlas_seed_peers = parser.get('blockstack', 'atlas_seeds')
# must be a CSV of host:port
hostports = filter( lambda x: len(x) > 0, atlas_seed_peers.split(",") )
for hp in hostports:
host, port = url_to_host_port( hp )
assert host is not None and port is not None
if parser.has_option('blockstack', 'atlasdb_path'):
atlasdb_path = parser.get('blockstack', 'atlasdb_path')
if parser.has_option('blockstack', 'atlas_blacklist'):
atlas_blacklist = parser.get('blockstack', 'atlas_blacklist')
# must be a CSV of host:port
hostports = filter( lambda x: len(x) > 0, atlas_blacklist.split(",") )
for hp in hostports:
host, port = url_to_host_port( hp )
assert host is not None and port is not None
if parser.has_option('blockstack', 'atlas_hostname'):
atlas_hostname = parser.get('blockstack', 'atlas_hostname')
if os.path.exists( announce_path ):
# load announcement list
with open( announce_path, "r" ) as f:
announce_text = f.readlines()
all_announcements = [ a.strip() for a in announce_text ]
unseen_announcements = []
# find announcements we haven't seen yet
for a in all_announcements:
if a not in ANNOUNCEMENTS:
unseen_announcements.append( a )
announcements = ",".join( unseen_announcements )
if zonefile_dir is not None and not os.path.exists( zonefile_dir ):
try:
os.makedirs( zonefile_dir, 0700 )
except:
pass
blockstack_opts = {
'rpc_port': rpc_port,
'email': contact_email,
'announcers': announcers,
'announcements': announcements,
'backup_frequency': backup_frequency,
'backup_max_age': backup_max_age,
'serve_zonefiles': serve_zonefiles,
'zonefile_storage_drivers': zonefile_storage_drivers,
"zonefile_storage_drivers_write": zonefile_storage_drivers_write,
'serve_profiles': serve_profiles,
'profile_storage_drivers': profile_storage_drivers,
"profile_storage_drivers_write": profile_storage_drivers_write,
'serve_data': serve_data,
'data_storage_drivers': data_storage_drivers,
"data_storage_drivers_write": data_storage_drivers_write,
'redirect_data': redirect_data,
'data_servers': data_servers,
'analytics_key': analytics_key,
'server_version': server_version,
'atlas': atlas_enabled,
'atlas_seeds': atlas_seed_peers,
'atlasdb_path': atlasdb_path,
'atlas_blacklist': atlas_blacklist,
'atlas_hostname': atlas_hostname,
'zonefiles': zonefile_dir,
}
# strip Nones
for (k, v) in blockstack_opts.items():
if v is None:
del blockstack_opts[k]
return blockstack_opts
def default_blockstack_opts( config_file=None ):
"""
Get our default blockstack opts from a config file
or from sane defaults.
"""
if config_file is None:
config_file = virtualchain.get_config_filename()
announce_path = get_announce_filename( virtualchain.get_working_dir() )
parser = SafeConfigParser()
parser.read( config_file )
blockstack_opts = {}
contact_email = None
announcers = "judecn.id,muneeb.id,shea256.id"
announcements = None
backup_frequency = 1008 # once a week; 10 minute block time
backup_max_age = 12096 # 12 weeks
rpc_port = RPC_SERVER_PORT
blockchain_proxy = False
serve_zonefiles = True
serve_profiles = False
zonefile_dir = None
analytics_key = None
zonefile_storage_drivers = "disk"
profile_storage_drivers = ""
server_version = None
if parser.has_section('blockstack'):
if parser.has_option('blockstack', 'backup_frequency'):
backup_frequency = int( parser.get('blockstack', 'backup_frequency'))
if parser.has_option('blockstack', 'backup_max_age'):
backup_max_age = int( parser.get('blockstack', 'backup_max_age') )
if parser.has_option('blockstack', 'email'):
contact_email = parser.get('blockstack', 'email')
if parser.has_option('blockstack', 'rpc_port'):
rpc_port = int(parser.get('blockstack', 'rpc_port'))
if parser.has_option('blockstack', 'blockchain_proxy'):
blockchain_proxy = parser.get('blockstack', 'blockchain_proxy')
if blockchain_proxy.lower() in ['1', 'yes', 'true', 'on']:
blockchain_proxy = True
else:
blockchain_proxy = False
if parser.has_option('blockstack', 'serve_zonefiles'):
serve_zonefiles = parser.get('blockstack', 'serve_zonefiles')
if serve_zonefiles.lower() in ['1', 'yes', 'true', 'on']:
serve_zonefiles = True
else:
serve_zonefiles = False
if parser.has_option('blockstack', 'serve_profiles'):
serve_profiles = parser.get('blockstack', 'serve_profiles')
if serve_profiles.lower() in ['1', 'yes', 'true', 'on']:
serve_profiles = True
else:
serve_profiles = False
if parser.has_option("blockstack", "zonefile_storage_drivers"):
zonefile_storage_drivers = parser.get("blockstack", "zonefile_storage_drivers")
if parser.has_option("blockstack", "profile_storage_drivers"):
profile_storage_drivers = parser.get("blockstack", "profile_storage_drivers")
if parser.has_option("blockstack", "zonefiles"):
zonefile_dir = parser.get("blockstack", "zonefiles")
if parser.has_option('blockstack', 'announcers'):
# must be a CSV of blockchain IDs
announcer_list_str = parser.get('blockstack', 'announcers')
announcer_list = announcer_list_str.split(",")
import scripts
# validate each one
valid = True
for bid in announcer_list:
if not scripts.is_name_valid( bid ):
log.error("Invalid blockchain ID '%s'" % bid)
valid = False
if valid:
announcers = ",".join(announcer_list)
if parser.has_option('blockstack', 'analytics_key'):
analytics_key = parser.get('blockstack', 'analytics_key')
if parser.has_option('blockstack', 'server_version'):
server_version = parser.get('blockstack', 'server_version')
if os.path.exists( announce_path ):
# load announcement list
with open( announce_path, "r" ) as f:
announce_text = f.readlines()
all_announcements = [ a.strip() for a in announce_text ]
unseen_announcements = []
# find announcements we haven't seen yet
for a in all_announcements:
if a not in ANNOUNCEMENTS:
unseen_announcements.append( a )
announcements = ",".join( unseen_announcements )
if zonefile_dir is not None and not os.path.exists( zonefile_dir ):
try:
os.makedirs( zonefile_dir, 0700 )
except:
pass
blockstack_opts = {
'rpc_port': rpc_port,
'email': contact_email,
'announcers': announcers,
'announcements': announcements,
'backup_frequency': backup_frequency,
'backup_max_age': backup_max_age,
'blockchain_proxy': blockchain_proxy,
'serve_zonefiles': serve_zonefiles,
'serve_profiles': serve_profiles,
'zonefile_storage_drivers': zonefile_storage_drivers,
'profile_storage_drivers': profile_storage_drivers,
'zonefiles': zonefile_dir,
'analytics_key': analytics_key,
'server_version': server_version
}
# strip Nones
for (k, v) in blockstack_opts.items():
if v is None:
del blockstack_opts[k]
return blockstack_opts
return []
inner.zonefile_count = 0
return inner
_zonefile_copy_progress = _zonefile_copy_progress()
# make sure we have the apppriate tools
tools = ['sqlite3']
for tool in tools:
rc = os.system("which {} > /dev/null".format(tool))
if rc != 0:
log.error("'{}' command not found".format(tool))
return False
working_dir = virtualchain.get_working_dir()
if not os.path.exists(working_dir):
log.error("No such directory {}".format(working_dir))
return False
if block_number is None:
# last backup
all_blocks = BlockstackDB.get_backup_blocks( virtualchain_hooks )
if len(all_blocks) == 0:
log.error("No backups available")
return False
block_number = max(all_blocks)
log.debug("Snapshot from block {}".format(block_number))
def get_indexing_lockfile(impl=None):
"""
Return path to the indexing lockfile
"""
return os.path.join( virtualchain.get_working_dir(impl=impl), "blockstack-server.indexing" )
def fast_sync_import(working_dir,
import_url,
public_keys=config.FAST_SYNC_PUBLIC_KEYS,
num_required=len(config.FAST_SYNC_PUBLIC_KEYS)):
"""
Fast sync import.
Verify the given fast-sync file from @import_path using @public_key, and then
uncompress it into @working_dir.
Verify that at least `num_required` public keys in `public_keys` signed.
NOTE: `public_keys` needs to be in the same order as the private keys that signed.
"""
# make sure we have the apppriate tools
tools = ['tar', 'bzip2', 'mv']
for tool in tools:
rc = os.system("which {} > /dev/null".format(tool))
if rc != 0:
log.error("'{}' command not found".format(tool))
return False
if working_dir is None:
working_dir = virtualchain.get_working_dir()
if not os.path.exists(working_dir):
log.error("No such directory {}".format(working_dir))
return False
# go get it
import_path = fast_sync_fetch(import_url)
if import_path is None:
log.error("Failed to fetch {}".format(import_url))
return False
# format: <signed bz2 payload> <sigb64> <sigb64 length (8 bytes hex)> ... <num signatures>
file_size = 0
try:
sb = os.stat(import_path)
file_size = sb.st_size
except Exception as e:
log.exception(e)
return False
num_signatures = 0
ptr = file_size
signatures = []
with open(import_path, 'r') as f:
info = fast_sync_inspect(f)
if 'error' in info:
log.error("Failed to inspect snapshot {}: {}".format(
import_path, info['error']))
return False
signatures = info['signatures']
ptr = info['payload_size']
# get the hash of the file
hash_hex = blockstack_client.storage.get_file_hash(f,
hashlib.sha256,
fd_len=ptr)
# validate signatures over the hash
log.debug("Verify {} bytes".format(ptr))
key_idx = 0
num_match = 0
for next_pubkey in public_keys:
for sigb64 in signatures:
valid = blockstack_client.keys.verify_digest(
hash_hex,
keylib.ECPublicKey(next_pubkey).to_hex(),
sigb64,
hashfunc=hashlib.sha256)
if valid:
num_match += 1
if num_match >= num_required:
break
log.debug("Public key {} matches {} ({})".format(
next_pubkey, sigb64, hash_hex))
signatures.remove(sigb64)
elif os.environ.get("BLOCKSTACK_TEST") == "1":
log.debug("Public key {} does NOT match {} ({})".format(
next_pubkey, sigb64, hash_hex))
# enough signatures?
if num_match < num_required:
log.error(
"Not enough signatures match (required {}, found {})".format(
num_required, num_match))
return False
# decompress
import_path = os.path.abspath(import_path)
cmd = "cd '{}' && tar xf '{}'".format(working_dir, import_path)
log.debug(cmd)
rc = os.system(cmd)
if rc != 0:
log.error("Failed to decompress. Exit code {}. Command: {}".format(
rc, cmd))
return False
# restore from backup
rc = blockstack_backup_restore(working_dir, None)
if not rc:
log.error("Failed to instantiate blockstack name database")
return False
# success!
log.debug("Restored to {}".format(working_dir))
return True
def store_announcement( announcement_hash, announcement_text, working_dir=None, force=False ):
"""
Store a new announcement locally, atomically.
"""
if working_dir is None:
working_dir = virtualchain.get_working_dir()
if not force:
# don't store unless we haven't seen it before
if announcement_hash in ANNOUNCEMENTS:
return
announce_filename = get_announce_filename( working_dir )
announce_filename_tmp = announce_filename + ".tmp"
announce_text = ""
announce_cleanup_list = []
# did we try (and fail) to store a previous announcement? If so, merge them all
if os.path.exists( announce_filename_tmp ):
log.debug("Merge announcement list %s" % announce_filename_tmp )
with open(announce_filename, "r") as f:
announce_text += f.read()
i = 1
failed_path = announce_filename_tmp + (".%s" % i)
while os.path.exists( failed_path ):
log.debug("Merge announcement list %s" % failed_paht )
with open(failed_path, "r") as f:
announce_text += f.read()
announce_cleanup_list.append( failed_path )
i += 1
failed_path = announce_filename_tmp + (".%s" % i)
announce_filename_tmp = failed_path
if os.path.exists( announce_filename ):
with open(announce_filename, "r" ) as f:
announce_text += f.read()
announce_text += ("\n%s\n" % announcement_hash)
# filter
if not force:
announcement_list = announce_text.split("\n")
unseen_announcements = filter( lambda a: a not in ANNOUNCEMENTS, announcement_list )
announce_text = "\n".join( unseen_announcements ).strip() + "\n"
log.debug("Store announcement hash to %s" % announce_filename )
with open(announce_filename_tmp, "w" ) as f:
f.write( announce_text )
f.flush()
# NOTE: rename doesn't remove the old file on Windows
if sys.platform == 'win32' and os.path.exists( announcement_filename_tmp ):
try:
os.unlink( announcement_filename_tmp )
except:
pass
try:
os.rename( announce_filename_tmp, announce_filename )
except:
log.error("Failed to save announcement %s to %s" % (announcement_hash, announce_filename ))
raise
# clean up
for tmp_path in announce_cleanup_list:
try:
os.unlink( tmp_path )
except:
pass
# put the announcement text
announcement_text_dir = os.path.join( working_dir, "announcements" )
if not os.path.exists( announcement_text_dir ):
try:
os.makedirs( announcement_text_dir )
except:
log.error("Failed to make directory %s" % announcement_text_dir )
raise
announcement_text_path = os.path.join( announcement_text_dir, "%s.txt" % announcement_hash )
try:
with open( announcement_text_path, "w" ) as f:
f.write( announcement_text )
except:
log.error("Failed to save announcement text to %s" % announcement_text_path )
raise
log.debug("Stored announcement to %s" % (announcement_text_path))
def get_working_path_file():
import talosvirtualchain
return os.path.join(get_working_dir(impl=talosvirtualchain), "tmp.dat")
def default_blockstack_opts( config_file=None, virtualchain_impl=None ):
"""
Get our default blockstack opts from a config file
or from sane defaults.
"""
if config_file is None:
config_file = virtualchain.get_config_filename()
announce_path = get_announce_filename( virtualchain.get_working_dir(impl=virtualchain_impl) )
parser = SafeConfigParser()
parser.read( config_file )
blockstack_opts = {}
contact_email = None
announcers = "judecn.id,muneeb.id,shea256.id"
announcements = None
backup_frequency = 144 # once a day; 10 minute block time
backup_max_age = 1008 # one week
rpc_port = RPC_SERVER_PORT
serve_zonefiles = True
serve_profiles = False
serve_data = False
zonefile_dir = os.path.join( os.path.dirname(config_file), "zonefiles")
analytics_key = None
zonefile_storage_drivers = "disk,dht"
zonefile_storage_drivers_write = "disk"
profile_storage_drivers = "disk"
profile_storage_drivers_write = "disk"
data_storage_drivers = "disk"
data_storage_drivers_write = "disk"
redirect_data = False
data_servers = None
server_version = None
atlas_enabled = True
atlas_seed_peers = "node.blockstack.org:%s" % RPC_SERVER_PORT
atlasdb_path = os.path.join( os.path.dirname(config_file), "atlas.db" )
atlas_blacklist = ""
atlas_hostname = socket.gethostname()
if parser.has_section('blockstack'):
if parser.has_option('blockstack', 'backup_frequency'):
backup_frequency = int( parser.get('blockstack', 'backup_frequency'))
if parser.has_option('blockstack', 'backup_max_age'):
backup_max_age = int( parser.get('blockstack', 'backup_max_age') )
if parser.has_option('blockstack', 'email'):
contact_email = parser.get('blockstack', 'email')
if parser.has_option('blockstack', 'rpc_port'):
rpc_port = int(parser.get('blockstack', 'rpc_port'))
if parser.has_option('blockstack', 'serve_zonefiles'):
serve_zonefiles = parser.get('blockstack', 'serve_zonefiles')
if serve_zonefiles.lower() in ['1', 'yes', 'true', 'on']:
serve_zonefiles = True
else:
serve_zonefiles = False
if parser.has_option('blockstack', 'serve_profiles'):
serve_profiles = parser.get('blockstack', 'serve_profiles')
if serve_profiles.lower() in ['1', 'yes', 'true', 'on']:
serve_profiles = True
else:
serve_profiles = False
if parser.has_option('blockstack', 'serve_data'):
serve_data = parser.get('blockstack', 'serve_data')
if serve_data.lower() in ['1', 'yes', 'true', 'on']:
serve_data = True
else:
serve_data = False
if parser.has_option("blockstack", "zonefile_storage_drivers"):
zonefile_storage_drivers = parser.get("blockstack", "zonefile_storage_drivers")
if parser.has_option("blockstack", "zonefile_storage_drivers_write"):
zonefile_storage_drivers_write = parser.get("blockstack", "zonefile_storage_drivers_write")
if parser.has_option("blockstack", "profile_storage_drivers"):
profile_storage_drivers = parser.get("blockstack", "profile_storage_drivers")
if parser.has_option("blockstack", "profile_storage_drivers_write"):
profile_storage_drivers_write = parser.get("blockstack", "profile_storage_drivers_write")
if parser.has_option("blockstack", "data_storage_drivers"):
data_storage_drivers = parser.get("blockstack", "data_storage_drivers")
if parser.has_option("blockstack", "data_storage_drivers_write"):
data_storage_drivers_write = parser.get("blockstack", "data_storage_drivers_write")
if parser.has_option("blockstack", "zonefiles"):
zonefile_dir = parser.get("blockstack", "zonefiles")
if parser.has_option('blockstack', 'redirect_data'):
redirect_data = parser.get('blockstack', 'redirect_data')
if redirect_data.lower() in ['1', 'yes', 'true', 'on']:
redirect_data = True
else:
redirect_data = False
if parser.has_option('blockstack', 'data_servers'):
data_servers = parser.get('blockstack', 'data_servers')
# must be a CSV of host:port
hostports = filter( lambda x: len(x) > 0, data_servers.split(",") )
for hp in hostports:
host, port = url_to_host_port( hp )
assert host is not None and port is not None
if parser.has_option('blockstack', 'announcers'):
# must be a CSV of blockchain IDs
announcer_list_str = parser.get('blockstack', 'announcers')
announcer_list = filter( lambda x: len(x) > 0, announcer_list_str.split(",") )
import scripts
# validate each one
valid = True
for bid in announcer_list:
if not scripts.is_name_valid( bid ):
log.error("Invalid blockchain ID '%s'" % bid)
valid = False
if valid:
announcers = ",".join(announcer_list)
if parser.has_option('blockstack', 'analytics_key'):
analytics_key = parser.get('blockstack', 'analytics_key')
if parser.has_option('blockstack', 'server_version'):
server_version = parser.get('blockstack', 'server_version')
if parser.has_option('blockstack', 'atlas'):
atlas_enabled = parser.get('blockstack', 'atlas')
if atlas_enabled.lower() in ['true', '1', 'enabled', 'enabled', 'on']:
atlas_enabled = True
else:
atlas_enabled = False
if parser.has_option('blockstack', 'atlas_seeds'):
atlas_seed_peers = parser.get('blockstack', 'atlas_seeds')
# must be a CSV of host:port
hostports = filter( lambda x: len(x) > 0, atlas_seed_peers.split(",") )
for hp in hostports:
host, port = url_to_host_port( hp )
assert host is not None and port is not None
if parser.has_option('blockstack', 'atlasdb_path'):
atlasdb_path = parser.get('blockstack', 'atlasdb_path')
if parser.has_option('blockstack', 'atlas_blacklist'):
atlas_blacklist = parser.get('blockstack', 'atlas_blacklist')
# must be a CSV of host:port
hostports = filter( lambda x: len(x) > 0, atlas_blacklist.split(",") )
for hp in hostports:
host, port = url_to_host_port( hp )
assert host is not None and port is not None
if parser.has_option('blockstack', 'atlas_hostname'):
atlas_hostname = parser.get('blockstack', 'atlas_hostname')
if os.path.exists( announce_path ):
# load announcement list
with open( announce_path, "r" ) as f:
announce_text = f.readlines()
all_announcements = [ a.strip() for a in announce_text ]
unseen_announcements = []
# find announcements we haven't seen yet
for a in all_announcements:
if a not in ANNOUNCEMENTS:
unseen_announcements.append( a )
announcements = ",".join( unseen_announcements )
if zonefile_dir is not None and not os.path.exists( zonefile_dir ):
try:
os.makedirs( zonefile_dir, 0700 )
except:
pass
blockstack_opts = {
'rpc_port': rpc_port,
'email': contact_email,
'announcers': announcers,
'announcements': announcements,
'backup_frequency': backup_frequency,
'backup_max_age': backup_max_age,
'serve_zonefiles': serve_zonefiles,
'zonefile_storage_drivers': zonefile_storage_drivers,
"zonefile_storage_drivers_write": zonefile_storage_drivers_write,
'serve_profiles': serve_profiles,
'profile_storage_drivers': profile_storage_drivers,
"profile_storage_drivers_write": profile_storage_drivers_write,
'serve_data': serve_data,
'data_storage_drivers': data_storage_drivers,
"data_storage_drivers_write": data_storage_drivers_write,
'redirect_data': redirect_data,
'data_servers': data_servers,
'analytics_key': analytics_key,
'server_version': server_version,
'atlas': atlas_enabled,
'atlas_seeds': atlas_seed_peers,
'atlasdb_path': atlasdb_path,
'atlas_blacklist': atlas_blacklist,
'atlas_hostname': atlas_hostname,
'zonefiles': zonefile_dir,
}
# strip Nones
for (k, v) in blockstack_opts.items():
if v is None:
del blockstack_opts[k]
return blockstack_opts
def default_blockstore_opts(config_file=None, testset=False):
"""
Get our default blockstore opts from a config file
or from sane defaults.
"""
if config_file is None:
config_file = virtualchain.get_config_filename()
testset_path = get_testset_filename(virtualchain.get_working_dir())
announce_path = get_announce_filename(virtualchain.get_working_dir())
parser = SafeConfigParser()
parser.read(config_file)
blockstore_opts = {}
tx_broadcaster = None
utxo_provider = None
testset_first_block = None
max_subsidy = 0
contact_email = None
announcers = "judecn.id,muneeb.id,shea256.id"
announcements = None
if parser.has_section('blockstore'):
if parser.has_option('blockstore', 'tx_broadcaster'):
tx_broadcaster = parser.get('blockstore', 'tx_broadcaster')
if parser.has_option('blockstore', 'utxo_provider'):
utxo_provider = parser.get('blockstore', 'utxo_provider')
if parser.has_option('blockstore', 'testset_first_block'):
testset_first_block = int(
parser.get('blockstore', 'testset_first_block'))
if parser.has_option('blockstore', 'max_subsidy'):
max_subsidy = int(parser.get('blockstore', 'max_subsidy'))
if parser.has_option('blockstore', 'email'):
contact_email = parser.get('blockstore', 'email')
if parser.has_option('blockstore', 'announcers'):
# must be a CSV of blockchain IDs
announcer_list_str = parser.get('blockstore', 'announcers')
announcer_list = announcer_list_str.split(",")
import scripts
# validate each one
valid = True
for bid in announcer_list:
if not scripts.is_name_valid(bid):
log.error("Invalid blockchain ID '%s'" % bid)
valid = False
if valid:
announcers = ",".join(announcer_list)
if os.path.exists(testset_path):
# testset file flag set
testset = True
if os.path.exists(announce_path):
# load announcement list
with open(announce_path, "r") as f:
announce_text = f.readlines()
all_announcements = [a.strip() for a in announce_text]
unseen_announcements = []
# find announcements we haven't seen yet
for a in all_announcements:
if a not in ANNOUNCEMENTS:
unseen_announcements.append(a)
announcements = ",".join(unseen_announcements)
blockstore_opts = {
'tx_broadcaster': tx_broadcaster,
'utxo_provider': utxo_provider,
'testset': testset,
'testset_first_block': testset_first_block,
'max_subsidy': max_subsidy,
'email': contact_email,
'announcers': announcers,
'announcements': announcements
}
# strip Nones
for (k, v) in blockstore_opts.items():
if v is None:
del blockstore_opts[k]
return blockstore_opts
def run_scenario(scenario, config_file):
"""
Run a test scenario:
* set up the virtualchain to use our mock UTXO provider and mock bitcoin blockchain
* seed it with the initial values in the wallet
* set the initial consensus hash
* run the scenario method
* run the check method
"""
mock_bitcoind_save_path = "/tmp/mock_bitcoind.dat"
if os.path.exists(mock_bitcoind_save_path):
try:
os.unlink(mock_bitcoind_save_path)
except:
pass
# use mock bitcoind
worker_env = {
# use mock_bitcoind to connect to bitcoind (but it has to import it in order to use it)
"VIRTUALCHAIN_MOD_CONNECT_BLOCKCHAIN": mock_bitcoind.__file__,
"MOCK_BITCOIND_SAVE_PATH": mock_bitcoind_save_path,
"BLOCKSTORE_TEST": "1",
}
if os.environ.get("PYTHONPATH", None) is not None:
worker_env["PYTHONPATH"] = os.environ["PYTHONPATH"]
virtualchain.setup_virtualchain(
blockstore_state_engine,
bitcoind_connection_factory=mock_bitcoind.connect_mock_bitcoind,
index_worker_env=worker_env,
)
# set up blockstore
# NOTE: utxo_opts encodes the mock-bitcoind options
blockstore_opts, bitcoin_opts, utxo_opts, dht_opts = blockstore.lib.configure(
config_file=config_file, interactive=False
)
# override multiprocessing options to ensure single-process behavior
utxo_opts["multiprocessing_num_procs"] = 1
utxo_opts["multiprocessing_num_blocks"] = 10
# pass along extra arguments
utxo_opts["save_file"] = mock_bitcoind_save_path
# save headers as well
utxo_opts["spv_headers_path"] = mock_bitcoind_save_path + ".spvheaders"
with open(utxo_opts["spv_headers_path"], "w") as f:
# write out "initial" headers, up to the first block
empty_header = ("00" * 81).decode("hex")
for i in xrange(0, blockstore.FIRST_BLOCK_MAINNET):
f.write(empty_header)
blockstored.set_bitcoin_opts(bitcoin_opts)
blockstored.set_utxo_opts(utxo_opts)
db = blockstored.get_state_engine()
bitcoind = mock_bitcoind.connect_mock_bitcoind(utxo_opts)
sync_virtualchain_upcall = lambda: virtualchain.sync_virtualchain(utxo_opts, bitcoind.getblockcount(), db)
mock_utxo = blockstore.lib.connect_utxo_provider(utxo_opts)
working_dir = virtualchain.get_working_dir()
# set up test environment
testlib.set_utxo_opts(utxo_opts)
testlib.set_utxo_client(mock_utxo)
testlib.set_bitcoind(bitcoind)
testlib.set_state_engine(db)
test_env = {
"sync_virtualchain_upcall": sync_virtualchain_upcall,
"working_dir": working_dir,
"bitcoind": bitcoind,
"bitcoind_save_path": mock_bitcoind_save_path,
}
# sync initial utxos
testlib.next_block(**test_env)
try:
os.unlink(mock_bitcoind_save_path)
except:
pass
# load the scenario into the mock blockchain and mock utxo provider
try:
scenario.scenario(scenario.wallets, **test_env)
except Exception, e:
log.exception(e)
traceback.print_exc()
log.error("Failed to run scenario '%s'" % scenario.__name__)
return False
