def get_db_state(disposition=DISPOSITION_RO):
impl = config.get_implementation()
if impl is None:
impl = sys.modules[__name__]
db_filename = config.get_db_filename(impl=impl)
lastblock_filename = config.get_lastblock_filename()
firstcheck = True
for path in [db_filename, lastblock_filename]:
if os.path.exists(path):
# have already create the db
firstcheck = False
if not firstcheck and not os.path.exists(lastblock_filename):
log.error("FATAL: no such file or directory: %s" % lastblock_filename)
# verify that it is well-formed if it exists
elif os.path.exists(lastblock_filename):
try:
with open(lastblock_filename, "r") as f:
int(f.read().strip())
except Exception, e:
log.error("FATAL: failed to parse: %s " % lastblock_filename)
log.exception(e)
os.abort()
def save( self, block_id, consensus_hash, pending_ops, backup=False ):
"""
Write out all state to the working directory.
Calls the implementation's 'db_save' method.
Return True on success
Return False on error
Raise exception if block_id represents a block
we've already processed.
"""
if block_id < self.lastblock:
raise Exception("Already processed up to block %s (got %s)" % (self.lastblock, block_id))
# stage data to temporary files
tmp_db_filename = (config.get_db_filename() + ".tmp")
tmp_snapshot_filename = (config.get_snapshots_filename() + ".tmp")
tmp_lastblock_filename = (config.get_lastblock_filename() + ".tmp")
with open(tmp_snapshot_filename, 'w') as f:
db_dict = {
'snapshots': self.consensus_hashes
}
f.write(json.dumps(db_dict))
f.flush()
# put this last...
with open(tmp_lastblock_filename, "w") as lastblock_f:
lastblock_f.write("%s" % block_id)
lastblock_f.flush()
rc = self.impl.db_save( block_id, consensus_hash, pending_ops, tmp_db_filename, db_state=self.state )
if not rc:
# failed to save
log.error("Implementation failed to save at block %s to %s" % (block_id, tmp_db_filename))
try:
os.unlink( tmp_lastblock_filename )
except:
pass
try:
os.unlink( tmp_snapshot_filename )
except:
pass
return False
rc = self.commit( backup=backup )
if not rc:
log.error("Failed to commit data at block %s. Rolling back." % block_id )
self.rollback()
return False
else:
self.lastblock = block_id
return True
def commit(self, backup=False, startup=False):
"""
Move all written but uncommitted data into place.
Return True on success
Return False on error (in which case the caller should rollback())
It is safe to call this method repeatedly until it returns True.
"""
if self.read_only:
log.error("FATAL: read-only")
os.abort()
tmp_db_filename = config.get_db_filename(impl=self.impl) + ".tmp"
tmp_snapshot_filename = config.get_snapshots_filename(impl=self.impl) + ".tmp"
tmp_lastblock_filename = config.get_lastblock_filename(impl=self.impl) + ".tmp"
if not os.path.exists(tmp_lastblock_filename) and (
os.path.exists(tmp_db_filename) or os.path.exists(tmp_snapshot_filename)):
# we did not successfully stage the write.
# rollback
log.error("Partial write detected. Not committing.")
return False
# basic sanity checks: don't overwrite the db if the file is zero bytes, or if we can't load it
if os.path.exists(tmp_db_filename):
db_dir = os.path.dirname(tmp_db_filename)
try:
dirfd = os.open(db_dir, os.O_DIRECTORY)
os.fsync(dirfd)
os.close(dirfd)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to sync directory %s" % db_dir)
traceback.print_stack()
os.abort()
sb = os.stat(tmp_db_filename)
if sb.st_size == 0:
log.error("Partial write detected: tried to overwrite with zero-sized db! Will rollback.")
return False
if startup:
# make sure we can load this
try:
with open(tmp_snapshot_filename, "r") as f:
db_txt = f.read()
db_json = json.loads(db_txt)
except:
log.error("Partial write detected: corrupt partially-committed db! Will rollback.")
return False
def commit( self, backup=False, startup=False ):
"""
Move all written but uncommitted data into place.
Return True on success
Return False on error (in which case the caller should rollback())
It is safe to call this method repeatedly until it returns True.
"""
if self.read_only:
log.error("FATAL: read-only")
os.abort()
tmp_db_filename = config.get_db_filename(impl=self.impl) + ".tmp"
tmp_snapshot_filename = config.get_snapshots_filename(impl=self.impl) + ".tmp"
tmp_lastblock_filename = config.get_lastblock_filename(impl=self.impl) + ".tmp"
if not os.path.exists( tmp_lastblock_filename ) and (os.path.exists(tmp_db_filename) or os.path.exists(tmp_snapshot_filename)):
# we did not successfully stage the write.
# rollback
log.error("Partial write detected. Not committing.")
return False
# basic sanity checks: don't overwrite the db if the file is zero bytes, or if we can't load it
if os.path.exists( tmp_db_filename ):
db_dir = os.path.dirname( tmp_db_filename )
try:
dirfd = os.open(db_dir, os.O_DIRECTORY)
os.fsync(dirfd)
os.close( dirfd )
except Exception, e:
log.exception(e)
log.error("FATAL: failed to sync directory %s" % db_dir)
traceback.print_stack()
os.abort()
sb = os.stat( tmp_db_filename )
if sb.st_size == 0:
log.error("Partial write detected: tried to overwrite with zero-sized db! Will rollback.")
return False
if startup:
# make sure we can load this
try:
with open(tmp_snapshot_filename, "r") as f:
db_txt = f.read()
db_json = json.loads(db_txt)
except:
log.error("Partial write detected: corrupt partially-committed db! Will rollback.")
return False
def rollback( self ):
"""
Roll back a pending write: blow away temporary files.
"""
tmp_db_filename = config.get_db_filename() + ".tmp"
tmp_snapshot_filename = config.get_snapshots_filename() + ".tmp"
tmp_lastblock_filename = config.get_lastblock_filename() + ".tmp"
for f in [tmp_db_filename, tmp_snapshot_filename, tmp_lastblock_filename]:
if os.path.exists( f ):
try:
os.unlink( f )
except:
log.error("Failed to unlink '%s'" % f )
pass
def save( self, block_id, consensus_hash, pending_ops, backup=False ):
"""
Write out all state to the working directory.
Calls the implementation's 'db_save' method to store any state for this block.
Calls the implementation's 'db_continue' method at the very end, to signal
to the implementation that all virtualchain state has been saved. This method
can return False, in which case, indexing stops
Return True on success
Return False if the implementation wants to exit.
Aborts on fatal error
"""
if self.read_only:
log.error("FATAL: read only")
traceback.print_stack()
os.abort()
if block_id < self.lastblock:
log.error("FATAL: Already processed up to block %s (got %s)" % (self.lastblock, block_id))
traceback.print_stack()
os.abort()
# stage data to temporary files
tmp_db_filename = (config.get_db_filename(impl=self.impl) + ".tmp")
tmp_snapshot_filename = (config.get_snapshots_filename(impl=self.impl) + ".tmp")
tmp_lastblock_filename = (config.get_lastblock_filename(impl=self.impl) + ".tmp")
try:
with open(tmp_snapshot_filename, 'w') as f:
db_dict = {
'snapshots': self.consensus_hashes
}
f.write(json.dumps(db_dict))
f.flush()
with open(tmp_lastblock_filename, "w") as lastblock_f:
lastblock_f.write("%s" % block_id)
lastblock_f.flush()
except Exception, e:
# failure to save is fatal
log.exception(e)
log.error("FATAL: Could not stage data for block %s" % block_id)
traceback.print_stack()
os.abort()
def save(self, block_id, consensus_hash, pending_ops, backup=False):
"""
Write out all state to the working directory.
Calls the implementation's 'db_save' method to store any state for this block.
Calls the implementation's 'db_continue' method at the very end, to signal
to the implementation that all virtualchain state has been saved. This method
can return False, in which case, indexing stops
Return True on success
Return False if the implementation wants to exit.
Aborts on fatal error
"""
if self.read_only:
log.error("FATAL: read only")
traceback.print_stack()
os.abort()
if block_id < self.lastblock:
log.error("FATAL: Already processed up to block %s (got %s)" % (self.lastblock, block_id))
traceback.print_stack()
os.abort()
# stage data to temporary files
tmp_db_filename = (config.get_db_filename(impl=self.impl) + ".tmp")
tmp_snapshot_filename = (config.get_snapshots_filename(impl=self.impl) + ".tmp")
tmp_lastblock_filename = (config.get_lastblock_filename(impl=self.impl) + ".tmp")
try:
with open(tmp_snapshot_filename, 'w') as f:
db_dict = {
'snapshots': self.consensus_hashes
}
f.write(json.dumps(db_dict))
f.flush()
with open(tmp_lastblock_filename, "w") as lastblock_f:
lastblock_f.write("%s" % block_id)
lastblock_f.flush()
except Exception, e:
# failure to save is fatal
log.exception(e)
log.error("FATAL: Could not stage data for block %s" % block_id)
traceback.print_stack()
os.abort()
def rollback( self ):
"""
Roll back a pending write: blow away temporary files.
"""
tmp_db_filename = config.get_db_filename() + ".tmp"
tmp_snapshot_filename = config.get_snapshots_filename() + ".tmp"
tmp_lastblock_filename = config.get_lastblock_filename() + ".tmp"
for f in [tmp_db_filename, tmp_snapshot_filename, tmp_lastblock_filename]:
if os.path.exists( f ):
try:
os.unlink( f )
except:
log.error("Failed to unlink '%s'" % f )
pass
def get_lastblock(self, lastblock_filename=None, impl=None):
"""
Get the last block
"""
if lastblock_filename is None:
if impl is None:
impl = self.impl
lastblock_filename = config.get_lastblock_filename()
if os.path.exists( lastblock_filename ):
try:
with open(lastblock_filename) as f:
lastblock_str = f.read().strip()
return int(lastblock_str)
except Exception, e:
log.error("Failed to read last block number at: %s" % lastblock_filename)
return None
def __init__(self, magic_bytes, opcodes, opfields, impl=None, state=None, initial_snapshots = {}, expected_snapshots={}, backup_frequency=None, backup_max_age=None, read_only=False):
self.consensus_hashes = initial_snapshots
self.pending_opts = defaultdict(list)
self.magic_bytes = magic_bytes
self.opcodes = opcodes[:]
self.opfields = copy.deepcopy(opfields)
self.state = state
self.impl = impl
self.lastblock = self.impl.get_first_block_id() - 1
self.pool = None
self.rejected = {}
self.expected_snapshots = expected_snapshots
self.backup_frequency = backup_frequency
self.backup_max_age = backup_max_age
self.read_only = read_only
firsttime = True
consensus_snapshots_filename = config.get_snapshots_filename(impl)
lastblock_filename = config.get_lastblock_filename(impl)
# check whether it is the first time
for fp in [consensus_snapshots_filename, lastblock_filename]:
if os.path.exists(fp):
firsttime = False
# Attempt to load the snapshots
if os.path.exists(consensus_snapshots_filename):
log.debug("Consensus snapshots at %s" % consensus_snapshots_filename)
try:
with open(consensus_snapshots_filename, "r") as f:
db_dict = json.loads(f.read())
assert 'snapshots' in db_dict
self.consensus_hashes = db_dict['snapshots']
except Exception, e:
log.error("FATAL: Failed to read consensus snapshots at %s" % (consensus_snapshots_filename))
log.exception(e)
os.abort()
def get_lastblock( self, lastblock_filename=None, impl=None, working_dir=None ):
"""
What was the last block processed?
Return the number on success
Return None on failure to read
"""
if lastblock_filename is None:
if impl is None:
impl = self.impl
lastblock_filename = config.get_lastblock_filename(impl=impl, working_dir=working_dir)
if os.path.exists( lastblock_filename ):
try:
with open(lastblock_filename, 'r') as f:
lastblock_str = f.read().strip()
return int(lastblock_str)
except Exception, e:
log.error("Failed to read last block number at '%s'" % lastblock_filename )
return None
def get_lastblock( self, lastblock_filename=None, impl=None ):
"""
What was the last block processed?
Return the number on success
Return None on failure to read
"""
if lastblock_filename is None:
if impl is None:
impl = self.impl
lastblock_filename = config.get_lastblock_filename(impl=impl)
if os.path.exists( lastblock_filename ):
try:
with open(lastblock_filename, 'r') as f:
lastblock_str = f.read().strip()
return int(lastblock_str)
except Exception, e:
log.error("Failed to read last block number at '%s'" % lastblock_filename )
return None
Return None if we fail to connect to bitcoind.
"""
start_block = config.get_first_block_id()
try:
current_block = int(bitcoind.getblockcount())
except Exception, e:
# TODO: reconnect on connection error
log.exception(e)
return None, None
# check our last known file
lastblock_file = config.get_lastblock_filename()
saved_block = 0
if os.path.isfile(lastblock_file):
with open(lastblock_file, 'r') as fin:
try:
saved_block = fin.read()
saved_block = int(saved_block)
except:
saved_block = 0
try:
os.unlink(lastblock_file)
except OSError, oe:
pass
def get_backup_paths( cls, block_id, impl ):
"""
Get the set of backup paths, given the virtualchain implementation module and block number
"""
backup_dir = os.path.join( config.get_working_dir(impl=impl), "backups" )
backup_paths = []
for p in [config.get_db_filename(impl=impl), config.get_snapshots_filename(impl=impl), config.get_lastblock_filename(impl=impl)]:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id))
backup_paths.append( backup_path )
return backup_paths
if startup:
# make sure we can load this
try:
with open(tmp_snapshot_filename, "r") as f:
db_txt = f.read()
db_json = json.loads(db_txt)
except:
log.error("Partial write detected: corrupt partially-committed db! Will rollback.")
return False
backup_time = int(time.time() * 1000000)
listing = []
listing.append(("lastblock", tmp_lastblock_filename, config.get_lastblock_filename(impl=self.impl)))
listing.append(("snapshots", tmp_snapshot_filename, config.get_snapshots_filename(impl=self.impl)))
listing.append(("db", tmp_db_filename, config.get_db_filename(impl=self.impl)))
for i in xrange(0, len(listing)):
file_type, tmp_filename, filename = listing[i]
dir_path = os.path.dirname(tmp_filename)
dirfd = None
try:
dirfd = os.open(dir_path, os.O_DIRECTORY)
os.fsync(dirfd)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to sync directory %s" % dir_path)
traceback.print_stack()
def commit(self, backup=False, startup=False):
"""
Move all written but uncommitted data into place.
Return True on success
Return False on error (in which case the caller should rollback())
It is safe to call this method repeatedly until it returns True.
"""
tmp_db_filename = config.get_db_filename() + ".tmp"
tmp_snapshot_filename = config.get_snapshots_filename() + ".tmp"
tmp_lastblock_filename = config.get_lastblock_filename() + ".tmp"
if not os.path.exists(tmp_lastblock_filename) and (
os.path.exists(tmp_db_filename)
or os.path.exists(tmp_snapshot_filename)):
# we did not successfully stage the write.
# rollback
log.error("Partial write detected. Not committing.")
return False
# basic sanity checks: don't overwrite the db if the file is zero bytes, or if we can't load it
if os.path.exists(tmp_db_filename):
sb = os.stat(tmp_db_filename)
if sb.st_size == 0:
log.error(
"Partial write detected: tried to overwrite with zero-sized db! Will rollback."
)
return False
if startup:
# make sure we can load this
try:
with open(tmp_snapshot_filename, "r") as f:
db_txt = f.read()
db_json = json.loads(db_txt)
except:
log.error(
"Partial write detected: corrupt partially-committed db! Will rollback."
)
return False
backup_time = int(time.time() * 1000000)
for tmp_filename, filename in zip( [tmp_lastblock_filename, tmp_snapshot_filename, tmp_db_filename], \
[config.get_lastblock_filename(), config.get_snapshots_filename(), config.get_db_filename()] ):
if not os.path.exists(tmp_filename):
continue
# commit our new lastblock, consensus hash set, and state engine data
try:
# NOTE: rename fails on Windows if the destination exists
if sys.platform == 'win32' and os.path.exists(filename):
try:
os.unlink(filename)
except:
pass
if not backup:
os.rename(tmp_filename, filename)
else:
shutil.copy(tmp_filename, filename)
os.rename(tmp_filename,
tmp_filename + (".%s" % backup_time))
except Exception, e:
log.exception(e)
return False
Get the range of block numbers that we need to fetch from the blockchain.
Return None if we fail to connect to bitcoind.
"""
start_block = config.get_first_block_id()
try:
current_block = int(bitcoind.getblockcount())
except Exception, e:
log.error(e)
return None, None
# check our last known file
lastblock_file = config.get_lastblock_filename()
saved_block = 0
if os.path.isfile(lastblock_file):
with open(lastblock_file, 'r') as fin:
try:
saved_block = fin.read()
saved_block = int(saved_block)
except:
saved_block = 0
try:
os.unlink(lastblock_file)
except OSError, oe:
pass
def commit( self, backup=False, startup=False ):
"""
Move all written but uncommitted data into place.
Return True on success
Return False on error (in which case the caller should rollback())
It is safe to call this method repeatedly until it returns True.
"""
tmp_db_filename = config.get_db_filename() + ".tmp"
tmp_snapshot_filename = config.get_snapshots_filename() + ".tmp"
tmp_lastblock_filename = config.get_lastblock_filename() + ".tmp"
if not os.path.exists( tmp_lastblock_filename ) and (os.path.exists(tmp_db_filename) or os.path.exists(tmp_snapshot_filename)):
# we did not successfully stage the write.
# rollback
log.error("Partial write detected. Not committing.")
return False
# basic sanity checks: don't overwrite the db if the file is zero bytes, or if we can't load it
if os.path.exists( tmp_db_filename ):
sb = os.stat( tmp_db_filename )
if sb.st_size == 0:
log.error("Partial write detected: tried to overwrite with zero-sized db! Will rollback.")
return False
if startup:
# make sure we can load this
try:
with open(tmp_snapshot_filename, "r") as f:
db_txt = f.read()
db_json = json.loads(db_txt)
except:
log.error("Partial write detected: corrupt partially-committed db! Will rollback.")
return False
backup_time = int(time.time() * 1000000)
for tmp_filename, filename in zip( [tmp_lastblock_filename, tmp_snapshot_filename, tmp_db_filename], \
[config.get_lastblock_filename(), config.get_snapshots_filename(), config.get_db_filename()] ):
if not os.path.exists( tmp_filename ):
continue
# commit our new lastblock, consensus hash set, and state engine data
try:
# NOTE: rename fails on Windows if the destination exists
if sys.platform == 'win32' and os.path.exists( filename ):
try:
os.unlink( filename )
except:
pass
if not backup:
os.rename( tmp_filename, filename )
else:
shutil.copy( tmp_filename, filename )
os.rename( tmp_filename, tmp_filename + (".%s" % backup_time))
except Exception, e:
log.exception(e)
return False
def get_backup_paths( cls, block_id, impl, working_dir=None ):
"""
Get the set of backup paths, given the virtualchain implementation module and block number
"""
backup_dir = os.path.join( config.get_working_dir(impl=impl, working_dir=working_dir), "backups" )
backup_paths = []
for p in [config.get_db_filename(impl=impl, working_dir=working_dir), config.get_snapshots_filename(impl=impl, working_dir=working_dir), config.get_lastblock_filename(impl=impl, working_dir=working_dir)]:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id))
backup_paths.append( backup_path )
return backup_paths
def backup_restore( cls, block_id, impl ):
"""
Restore from a backup, given the virutalchain implementation module and block number
"""
backup_dir = os.path.join( config.get_working_dir(impl=impl), "backups" )
backup_paths = cls.get_backup_paths( block_id, impl )
for p in backup_paths:
assert os.path.exists( p ), "No such backup file: %s" % backup_paths
for p in [config.get_db_filename(impl=impl), config.get_snapshots_filename(impl=impl), config.get_lastblock_filename(impl=impl)]:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id))
log.debug("Restoring '%s' to '%s'" % (backup_path, p))
shutil.copy( backup_path, p )
return True
def __init__(self, magic_bytes, opcodes, opfields, impl=None, state=None, initial_snapshots={} ):
"""
Construct a state engine client, optionally from locally-cached
state and the set of previously-calculated consensus
hashes for each block.
This class will be fed a sequence of sets of transactions, grouped by block
and ordered by block ID, that each contain an OP_RETURN. The nulldata
assocated with the OP_RETURN will be parsed, checked, logged, and
committed by the implementation (impl). The implementation decides exactly
what each of these mean; this class simply feeds it the transactions
in the order they appeared on the blockchain.
This class looks for OP_RETURN data that starts with the byte sequence in magic_bytes,
and then only select those which start with magic_bytes + op, where op is an
opcode byte in opcodes. Magic bytes can be of variable length, but it should
be specific to this virtual chain.
Expected OP_RETURN data format:
0 M M+1 len(OP_RETURN)-M-1
|-----|--|------------------------|
magic op payload
The job of the implementation is to translate the above data, plus anything else it
can earn from the previously-parsed transactions and from other sources, into a
dictionary of (field: value) tuples that constitute an operation.
@magic_bytes: the `magic` field above.
@opcodes: the list of possible values for the `op` field.
@opfields: a dictionary that maps each `op` to a list of field names.
The caller may supply an optional argument called 'state', which will be
passed into each implementation method. It is meant to preserve implementation-
specific state--in particular, whatever state the implementation expects to be
present.
"""
self.consensus_hashes = initial_snapshots
self.pending_ops = defaultdict(list)
self.magic_bytes = magic_bytes
self.opcodes = opcodes[:]
self.opfields = copy.deepcopy(opfields)
self.state = state
self.impl = impl
self.lastblock = self.impl.get_first_block_id() - 1
self.pool = None
self.rejected = {}
consensus_snapshots_filename = config.get_snapshots_filename()
lastblock_filename = config.get_lastblock_filename()
# if we crashed during a commit, try to finish
rc = self.commit( startup=True )
if not rc:
log.error("Failed to commit partial data. Rolling back.")
self.rollback()
# attempt to load the snapshots
if os.path.exists( consensus_snapshots_filename ):
try:
with open(consensus_snapshots_filename, 'r') as f:
db_dict = json.loads(f.read())
if 'snapshots' in db_dict:
self.consensus_hashes = db_dict['snapshots']
except Exception, e:
log.error("Failed to read consensus snapshots at '%s'" % consensus_snapshots_filename )
raise e
def backup_restore( cls, block_id, impl, working_dir=None ):
"""
Restore from a backup, given the virutalchain implementation module and block number
"""
backup_dir = os.path.join( config.get_working_dir(impl=impl, working_dir=working_dir), "backups" )
backup_paths = cls.get_backup_paths( block_id, impl, working_dir=working_dir )
for p in backup_paths:
assert os.path.exists( p ), "No such backup file: %s" % backup_paths
for p in [config.get_db_filename(impl=impl, working_dir=working_dir), config.get_snapshots_filename(impl=impl, working_dir=working_dir), config.get_lastblock_filename(impl=impl, working_dir=working_dir)]:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id))
log.debug("Restoring '%s' to '%s'" % (backup_path, p))
shutil.copy( backup_path, p )
return True
def make_backups( self, block_id ):
"""
If we're doing backups on a regular basis, then
carry them out here if it is time to do so.
This method does nothing otherwise.
Abort on failure
"""
# make a backup?
if self.backup_frequency is not None:
if (block_id % self.backup_frequency) == 0:
backup_dir = os.path.join( config.get_working_dir(impl=self.impl), "backups" )
if not os.path.exists(backup_dir):
try:
os.makedirs(backup_dir)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to make backup directory '%s'" % backup_dir)
traceback.print_stack()
os.abort()
for p in [config.get_db_filename(impl=self.impl), config.get_snapshots_filename(impl=self.impl), config.get_lastblock_filename(impl=self.impl)]:
if os.path.exists(p):
try:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % (block_id - 1)))
if not os.path.exists( backup_path ):
shutil.copy( p, backup_path )
else:
log.error("Will not overwrite '%s'" % backup_path)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to back up '%s'" % p)
traceback.print_stack()
os.abort()
def make_backups( self, block_id, working_dir=None ):
"""
If we're doing backups on a regular basis, then
carry them out here if it is time to do so.
This method does nothing otherwise.
Abort on failure
"""
# make a backup?
if self.backup_frequency is not None:
if (block_id % self.backup_frequency) == 0:
backup_dir = os.path.join( config.get_working_dir(impl=self.impl, working_dir=working_dir), "backups" )
if not os.path.exists(backup_dir):
try:
os.makedirs(backup_dir)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to make backup directory '%s'" % backup_dir)
traceback.print_stack()
os.abort()
for p in [config.get_db_filename(impl=self.impl, working_dir=working_dir), config.get_snapshots_filename(impl=self.impl, working_dir=working_dir), config.get_lastblock_filename(impl=self.impl, working_dir=working_dir)]:
if os.path.exists(p):
try:
pbase = os.path.basename(p)
backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % (block_id - 1)))
if not os.path.exists( backup_path ):
shutil.copy( p, backup_path )
else:
log.error("Will not overwrite '%s'" % backup_path)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to back up '%s'" % p)
traceback.print_stack()
os.abort()
def __init__(self, magic_bytes, opcodes, impl=None, state=None, op_order=None, initial_snapshots={} ):
"""
Construct a state engine client, optionally from locally-cached
state and the set of previously-calculated consensus
hashes for each block.
This class will be fed a sequence of sets of transactions, grouped by block
and ordered by block ID, that each contain an OP_RETURN. The nulldata
assocated with the OP_RETURN will be parsed, checked, logged, and
committed by the implementation (impl). The implementation decides exactly
what each of these mean; this class simply feeds it the transactions
in order.
This class looks for OP_RETURN data that starts with the byte sequence in magic_bytes,
and then only select those which start with magic_bytes + op, where op is an
opcode byte in opcodes. Magic bytes can be of variable length, but it should
be specific to this virtual chain.
Expected OP_RETURN data format:
0 M M+1 40-M-1
|-----|--|------------------------|
magic op payload
The caller may supply an optional argument called 'state', which will be
passed into each implementation method. It is meant to preserve implementation-
specific state--in particular, whatever state the implementation expects to be
present.
The caller may also specify the order in which each type of operation is
processed, by passing a list of opcodes in op_order.
"""
self.consensus_hashes = initial_snapshots
self.pending_ops = defaultdict(list)
self.magic_bytes = magic_bytes
self.opcodes = opcodes[:]
self.state = state
self.op_order = op_order
self.impl = impl
self.lastblock = self.impl.get_first_block_id() - 1
self.pool = None
self.rejected = {}
if self.op_order is None:
self.op_order = self.impl.get_op_processing_order()[:]
if self.op_order is None:
self.op_order = opcodes
# there's always a 'final' operation type, to be processed last
self.op_order.append('virtualchain_final')
consensus_snapshots_filename = config.get_snapshots_filename()
lastblock_filename = config.get_lastblock_filename()
# if we crashed during a commit, try to finish
rc = self.commit( startup=True )
if not rc:
log.error("Failed to commit partial data. Rolling back.")
self.rollback()
# attempt to load the snapshots
if os.path.exists( consensus_snapshots_filename ):
try:
with open(consensus_snapshots_filename, 'r') as f:
db_dict = json.loads(f.read())
if 'snapshots' in db_dict:
self.consensus_hashes = db_dict['snapshots']
except Exception, e:
log.error("Failed to read consensus snapshots at '%s'" % consensus_snapshots_filename )
raise e
def __init__(self,
magic_bytes,
opcodes,
opfields,
impl=None,
state=None,
initial_snapshots={}):
"""
Construct a state engine client, optionally from locally-cached
state and the set of previously-calculated consensus
hashes for each block.
This class will be fed a sequence of sets of transactions, grouped by block
and ordered by block ID, that each contain an OP_RETURN. The nulldata
assocated with the OP_RETURN will be parsed, checked, logged, and
committed by the implementation (impl). The implementation decides exactly
what each of these mean; this class simply feeds it the transactions
in the order they appeared on the blockchain.
This class looks for OP_RETURN data that starts with the byte sequence in magic_bytes,
and then only select those which start with magic_bytes + op, where op is an
opcode byte in opcodes. Magic bytes can be of variable length, but it should
be specific to this virtual chain.
Expected OP_RETURN data format:
0 M M+1 len(OP_RETURN)-M-1
|-----|--|------------------------|
magic op payload
The job of the implementation is to translate the above data, plus anything else it
can earn from the previously-parsed transactions and from other sources, into a
dictionary of (field: value) tuples that constitute an operation.
@magic_bytes: the `magic` field above.
@opcodes: the list of possible values for the `op` field.
@opfields: a dictionary that maps each `op` to a list of field names.
The caller may supply an optional argument called 'state', which will be
passed into each implementation method. It is meant to preserve implementation-
specific state--in particular, whatever state the implementation expects to be
present.
"""
self.consensus_hashes = initial_snapshots
self.pending_ops = defaultdict(list)
self.magic_bytes = magic_bytes
self.opcodes = opcodes[:]
self.opfields = copy.deepcopy(opfields)
self.state = state
self.impl = impl
self.lastblock = self.impl.get_first_block_id() - 1
self.pool = None
self.rejected = {}
consensus_snapshots_filename = config.get_snapshots_filename()
lastblock_filename = config.get_lastblock_filename()
# if we crashed during a commit, try to finish
rc = self.commit(startup=True)
if not rc:
log.error("Failed to commit partial data. Rolling back.")
self.rollback()
# attempt to load the snapshots
if os.path.exists(consensus_snapshots_filename):
try:
with open(consensus_snapshots_filename, 'r') as f:
db_dict = json.loads(f.read())
if 'snapshots' in db_dict:
self.consensus_hashes = db_dict['snapshots']
except Exception, e:
log.error("Failed to read consensus snapshots at '%s'" %
consensus_snapshots_filename)
raise e
def __init__(self,
magic_bytes,
opcodes,
impl=None,
state=None,
op_order=None,
initial_snapshots={}):
"""
Construct a state engine client, optionally from locally-cached
state and the set of previously-calculated consensus
hashes for each block.
This class will be fed a sequence of sets of transactions, grouped by block
and ordered by block ID, that each contain an OP_RETURN. The nulldata
assocated with the OP_RETURN will be parsed, checked, logged, and
committed by the implementation (impl). The implementation decides exactly
what each of these mean; this class simply feeds it the transactions
in order.
This class looks for OP_RETURN data that starts with the byte sequence in magic_bytes,
and then only select those which start with magic_bytes + op, where op is an
opcode byte in opcodes. Magic bytes can be of variable length, but it should
be specific to this virtual chain.
Expected OP_RETURN data format:
0 M M+1 40-M-1
|-----|--|------------------------|
magic op payload
The caller may supply an optional argument called 'state', which will be
passed into each implementation method. It is meant to preserve implementation-
specific state--in particular, whatever state the implementation expects to be
present.
The caller may also specify the order in which each type of operation is
processed, by passing a list of opcodes in op_order.
"""
self.consensus_hashes = initial_snapshots
self.pending_ops = defaultdict(list)
self.magic_bytes = magic_bytes
self.opcodes = opcodes[:]
self.state = state
self.op_order = op_order
self.impl = impl
self.lastblock = self.impl.get_first_block_id() - 1
self.pool = None
self.rejected = {}
if self.op_order is None:
self.op_order = self.impl.get_op_processing_order()[:]
if self.op_order is None:
self.op_order = opcodes
# there's always a 'final' operation type, to be processed last
self.op_order.append('virtualchain_final')
consensus_snapshots_filename = config.get_snapshots_filename()
lastblock_filename = config.get_lastblock_filename()
# if we crashed during a commit, try to finish
rc = self.commit(startup=True)
if not rc:
log.error("Failed to commit partial data. Rolling back.")
self.rollback()
# attempt to load the snapshots
if os.path.exists(consensus_snapshots_filename):
try:
with open(consensus_snapshots_filename, 'r') as f:
db_dict = json.loads(f.read())
if 'snapshots' in db_dict:
self.consensus_hashes = db_dict['snapshots']
except Exception, e:
log.error("Failed to read consensus snapshots at '%s'" %
consensus_snapshots_filename)
raise e
def save(self, block_id, consensus_hash, pending_ops, backup=False):
"""
Write out all state to the working directory.
Calls the implementation's 'db_save' method.
Return True on success
Return False on error
Raise exception if block_id represents a block
we've already processed.
"""
if block_id < self.lastblock:
raise Exception("Already processed up to block %s (got %s)" %
(self.lastblock, block_id))
# stage data to temporary files
tmp_db_filename = (config.get_db_filename() + ".tmp")
tmp_snapshot_filename = (config.get_snapshots_filename() + ".tmp")
tmp_lastblock_filename = (config.get_lastblock_filename() + ".tmp")
with open(tmp_snapshot_filename, 'w') as f:
db_dict = {'snapshots': self.consensus_hashes}
f.write(json.dumps(db_dict))
f.flush()
# put this last...
with open(tmp_lastblock_filename, "w") as lastblock_f:
lastblock_f.write("%s" % block_id)
lastblock_f.flush()
rc = self.impl.db_save(block_id,
consensus_hash,
pending_ops,
tmp_db_filename,
db_state=self.state)
if not rc:
# failed to save
log.error("Implementation failed to save at block %s to %s" %
(block_id, tmp_db_filename))
try:
os.unlink(tmp_lastblock_filename)
except:
pass
try:
os.unlink(tmp_snapshot_filename)
except:
pass
return False
rc = self.commit(backup=backup)
if not rc:
log.error("Failed to commit data at block %s. Rolling back." %
block_id)
self.rollback()
return False
else:
self.lastblock = block_id
return True
if startup:
# make sure we can load this
try:
with open(tmp_snapshot_filename, "r") as f:
db_txt = f.read()
db_json = json.loads(db_txt)
except:
log.error("Partial write detected: corrupt partially-committed db! Will rollback.")
return False
backup_time = int(time.time() * 1000000)
listing = []
listing.append( ("lastblock", tmp_lastblock_filename, config.get_lastblock_filename(impl=self.impl)) )
listing.append( ("snapshots", tmp_snapshot_filename, config.get_snapshots_filename(impl=self.impl)) )
listing.append( ("db", tmp_db_filename, config.get_db_filename(impl=self.impl)) )
for i in xrange(0, len(listing)):
file_type, tmp_filename, filename = listing[i]
dir_path = os.path.dirname( tmp_filename )
dirfd = None
try:
dirfd = os.open(dir_path, os.O_DIRECTORY)
os.fsync(dirfd)
except Exception, e:
log.exception(e)
log.error("FATAL: failed to sync directory %s" % dir_path)
traceback.print_stack()
