def get_backup_blocks( cls, impl ):
"""
Get the set of block IDs that were backed up
"""
ret = []
backup_dir = os.path.join( config.get_working_dir(impl=impl), "backups" )
if not os.path.exists(backup_dir):
return []
for name in os.listdir( backup_dir ):
if ".bak." not in name:
continue
suffix = name.split(".bak.")[-1]
try:
block_id = int(suffix)
except:
continue
# must exist...
backup_paths = cls.get_backup_paths( block_id, impl )
for p in backup_paths:
if not os.path.exists(p):
# doesn't exist
block_id = None
continue
if block_id is not None:
# have backup at this block
ret.append(block_id)
return ret
def get_backup_blocks( cls, impl, working_dir=None ):
"""
Get the set of block IDs that were backed up
"""
ret = []
backup_dir = os.path.join( config.get_working_dir(impl=impl, working_dir=working_dir), "backups" )
if not os.path.exists(backup_dir):
return []
for name in os.listdir( backup_dir ):
if ".bak." not in name:
continue
suffix = name.split(".bak.")[-1]
try:
block_id = int(suffix)
except:
continue
# must exist...
backup_paths = cls.get_backup_paths( block_id, impl, working_dir=working_dir )
for p in backup_paths:
if not os.path.exists(p):
# doesn't exist
block_id = None
continue
if block_id is not None:
# have backup at this block
ret.append(block_id)
return ret
def get_backup_paths( cls, block_id, impl, working_dir=None ):
"""
Get the set of backup paths, given the virtualchain implementation module and block number
"""
backup_dir = os.path.join( config.get_working_dir(impl=impl, working_dir=working_dir), "backups" )
backup_paths = []
for p in [config.get_db_filename(impl=impl, working_dir=working_dir), config.get_snapshots_filename(impl=impl, working_dir=working_dir), config.get_lastblock_filename(impl=impl, working_dir=working_dir)]:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id))
backup_paths.append( backup_path )
return backup_paths
def get_backup_paths( cls, block_id, impl ):
"""
Get the set of backup paths, given the virtualchain implementation module and block number
"""
backup_dir = os.path.join( config.get_working_dir(impl=impl), "backups" )
backup_paths = []
for p in [config.get_db_filename(impl=impl), config.get_snapshots_filename(impl=impl), config.get_lastblock_filename(impl=impl)]:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id))
backup_paths.append( backup_path )
return backup_paths
def main(args):
with open(config.get_json_path(), 'rb') as fp:
patches = data.parse_json(fp.read())
working_dir = config.get_working_dir()
commit = gitcmd.get_sha1(config.get_master_branch())
bots = args.bots
if not bots:
bots = config.get_buildbots()
for bot in bots:
q = config.get_buildbot_query(bot)
run_bot(patches, working_dir, commit, bot, q)
def main(args):
with open(config.get_json_path(), 'rb') as fp:
patches = data.parse_json(fp.read())
working_dir = config.get_working_dir()
commit = gitcmd.get_sha1(config.get_master_branch())
bots = args.bots
if not bots:
bots = config.get_buildbots()
for bot in bots:
q = config.get_buildbot_query(bot)
run_bot(patches, working_dir, commit, bot, q)
def backup_restore( cls, block_id, impl, working_dir=None ):
"""
Restore from a backup, given the virutalchain implementation module and block number
"""
backup_dir = os.path.join( config.get_working_dir(impl=impl, working_dir=working_dir), "backups" )
backup_paths = cls.get_backup_paths( block_id, impl, working_dir=working_dir )
for p in backup_paths:
assert os.path.exists( p ), "No such backup file: %s" % backup_paths
for p in [config.get_db_filename(impl=impl, working_dir=working_dir), config.get_snapshots_filename(impl=impl, working_dir=working_dir), config.get_lastblock_filename(impl=impl, working_dir=working_dir)]:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id))
log.debug("Restoring '%s' to '%s'" % (backup_path, p))
shutil.copy( backup_path, p )
return True
def backup_restore( cls, block_id, impl ):
"""
Restore from a backup, given the virutalchain implementation module and block number
"""
backup_dir = os.path.join( config.get_working_dir(impl=impl), "backups" )
backup_paths = cls.get_backup_paths( block_id, impl )
for p in backup_paths:
assert os.path.exists( p ), "No such backup file: %s" % backup_paths
for p in [config.get_db_filename(impl=impl), config.get_snapshots_filename(impl=impl), config.get_lastblock_filename(impl=impl)]:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id))
log.debug("Restoring '%s' to '%s'" % (backup_path, p))
shutil.copy( backup_path, p )
return True
def clear_old_backups( self, block_id, working_dir=None ):
"""
If we limit the number of backups we make, then clean out old ones
older than block_id - backup_max_age (given in the constructor)
This method does nothing otherwise.
"""
if self.backup_max_age is None:
# never delete backups
return
# find old backups
backup_dir = os.path.join( config.get_working_dir(impl=self.impl, working_dir=working_dir), "backups" )
if not os.path.exists(backup_dir):
return
backups = os.listdir( backup_dir )
for backup_name in backups:
if backup_name in [".", ".."]:
continue
backup_path = os.path.join(backup_dir, backup_name)
backup_block = None
try:
backup_block = int(backup_path.split(".")[-1])
except:
# not a backup file
log.info("Skipping non-backup '%s'" % backup_path)
if not backup_path.endswith( ".bak.%s" % backup_block ):
# not a backup file
log.info("Skipping non-backup '%s'" % backup_path)
continue
if backup_block + self.backup_max_age < block_id:
# dead
log.info("Removing old backup '%s'" % backup_path)
try:
os.unlink(backup_path)
except:
pass
def clear_old_backups( self, block_id ):
"""
If we limit the number of backups we make, then clean out old ones
older than block_id - backup_max_age (given in the constructor)
This method does nothing otherwise.
"""
if self.backup_max_age is None:
# never delete backups
return
# find old backups
backup_dir = os.path.join( config.get_working_dir(impl=self.impl), "backups" )
if not os.path.exists(backup_dir):
return
backups = os.listdir( backup_dir )
for backup_name in backups:
if backup_name in [".", ".."]:
continue
backup_path = os.path.join(backup_dir, backup_name)
backup_block = None
try:
backup_block = int(backup_path.split(".")[-1])
except:
# not a backup file
log.info("Skipping non-backup '%s'" % backup_path)
if not backup_path.endswith( ".bak.%s" % backup_block ):
# not a backup file
log.info("Skipping non-backup '%s'" % backup_path)
continue
if backup_block + self.backup_max_age < block_id:
# dead
log.info("Removing old backup '%s'" % backup_path)
try:
os.unlink(backup_path)
except:
pass
def make_backups( self, block_id, working_dir=None ):
"""
If we're doing backups on a regular basis, then
carry them out here if it is time to do so.
This method does nothing otherwise.
Abort on failure
"""
# make a backup?
if self.backup_frequency is not None:
if (block_id % self.backup_frequency) == 0:
backup_dir = os.path.join( config.get_working_dir(impl=self.impl, working_dir=working_dir), "backups" )
if not os.path.exists(backup_dir):
try:
os.makedirs(backup_dir)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to make backup directory '%s'" % backup_dir)
traceback.print_stack()
os.abort()
for p in [config.get_db_filename(impl=self.impl, working_dir=working_dir), config.get_snapshots_filename(impl=self.impl, working_dir=working_dir), config.get_lastblock_filename(impl=self.impl, working_dir=working_dir)]:
if os.path.exists(p):
try:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % (block_id - 1)))
if not os.path.exists( backup_path ):
shutil.copy( p, backup_path )
else:
log.error("Will not overwrite '%s'" % backup_path)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to back up '%s'" % p)
traceback.print_stack()
os.abort()
def make_backups( self, block_id ):
"""
If we're doing backups on a regular basis, then
carry them out here if it is time to do so.
This method does nothing otherwise.
Abort on failure
"""
# make a backup?
if self.backup_frequency is not None:
if (block_id % self.backup_frequency) == 0:
backup_dir = os.path.join( config.get_working_dir(impl=self.impl), "backups" )
if not os.path.exists(backup_dir):
try:
os.makedirs(backup_dir)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to make backup directory '%s'" % backup_dir)
traceback.print_stack()
os.abort()
for p in [config.get_db_filename(impl=self.impl), config.get_snapshots_filename(impl=self.impl), config.get_lastblock_filename(impl=self.impl)]:
if os.path.exists(p):
try:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % (block_id - 1)))
if not os.path.exists( backup_path ):
shutil.copy( p, backup_path )
else:
log.error("Will not overwrite '%s'" % backup_path)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to back up '%s'" % p)
traceback.print_stack()
os.abort()
def run(self, *args, task_id=None, **kwargs):
"""
Run the job.
This is an override of `Task.run` which is the method
that actually gets called by Celery each time a task
in processed. It wraps `self.do()` to handle
logging, exceptions, termination etc.
Most jobs need to operate within a project's working directory
and project `File` paths are always relative to those.
To avoid code repetition and potential errors this method requires
that a project id is supplied and changes into the working
directory of that project.
"""
project = kwargs.get("project")
if project is None:
raise ValueError("Project number must be provided as an argument!")
current_dir = os.getcwd()
working_dir = get_working_dir(project)
ensure_dir(working_dir)
self.begin(task_id)
try:
os.chdir(working_dir)
result = self.do(*args, **kwargs)
return self.success(result)
except SoftTimeLimitExceeded:
return self.terminated()
except Ignore as exc:
raise exc
except Exception as exc:
raise self.failure(exc)
finally:
os.chdir(current_dir)
def run_scenario(scenario,
config_file,
client_config_file,
interactive=False,
blocktime=10):
"""
* set up the virtualchain to use mock UTXO provider and mock bitcoin blockchain
* seed it with the intial value in the wallet
* set the intial consensus hash
* start the api server
* run the scenario method
* run the check method
"""
virtualchain_working_dir = os.environ["VIRTUALCHAIN_WORKING_DIR"]
spv_header_path = os.path.join(virtualchain_working_dir, "spv_headers.dat")
virtualchain.setup_virtualchain(state_engine)
db = state_engine.get_db_state(disposition=state_engine.DISPOSITION_RW)
log.info("Connect to the bitcoind ")
bitcoind = bitcoin_regtest_connect(bitcoin_regtest_opts())
working_dir = get_working_dir()
utxo_opts = {}
# Start the pinger
pinger = Pinger()
pinger.start()
#set up the environment
testlib.set_utxo_opts(utxo_opts)
testlib.set_bitcoind(bitcoind)
testlib.set_state_engine(db)
test_env = {
"sync_virtualchain_upcall":
lambda: sync_virtualchain_upcall(zonefilemanage_opts=None,
need_db_refresh=False),
"next_block_upcall":
bitcoin_regtest_next_block,
"working_dir":
working_dir,
"bitcoind":
bitcoind,
"bitcoind_opts":
bitcoin_regtest_opts(),
"spv_header_path":
spv_header_path
}
# Sync initial utxos
testlib.next_block(**test_env)
# Load the scenario into the mock blockchain and mock utxo provider
try:
rc = scenario.scenario(scenario.wallets, **test_env)
except Exception, e:
log.exception(e)
traceback.print_exc()
