class ChainDb(object): def __init__(self, settings, datadir, log, mempool, netmagic, readonly=False, fast_dbm=False, compression=False): self.settings = settings self.log = log self.mempool = mempool self.readonly = readonly self.netmagic = netmagic self.fast_dbm = fast_dbm self.blk_cache = Cache(1000) self.orphans = {} self.orphan_deps = {} self.compress_on_write = compression # LevelDB to hold: # tx:* transaction outputs # misc:* state # height:* list of blocks at height h # blkmeta:* block metadata # blocks:* block seek point in stream self.blk_write = io.BufferedWriter( io.FileIO(datadir + '/blocks.dat', 'ab')) self.blk_read = io.BufferedReader( io.FileIO(datadir + '/blocks.dat', 'rb')) self.db = leveldb.LevelDB(datadir + '/leveldb') try: self.db.Get('misc:height') except KeyError: self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE") batch = leveldb.WriteBatch() batch.Put('misc:height', str(-1)) batch.Put('misc:msg_start', self.netmagic.msg_start) batch.Put('misc:tophash', ser_uint256(0L)) batch.Put('misc:total_work', hex(0L)) self.db.Write(batch) try: start = self.db.Get('misc:msg_start') if start != self.netmagic.msg_start: raise KeyError except KeyError: self.log.write( "Database magic number mismatch. Data corruption or incorrect network?" ) raise RuntimeError def puttxidx(self, txhash, txidx, batch=None): ser_txhash = ser_uint256(txhash) try: self.db.Get('tx:' + ser_txhash) old_txidx = self.gettxidx(txhash) self.log.write( "WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x" % (txhash, self.getheight(), old_txidx.blkhash, old_txidx.spentmask, txidx.blkhash)) except KeyError: pass batch = self.db if batch is not None else batch batch.Put('tx:' + ser_txhash, hex(txidx.blkhash) + ' ' + hex(txidx.spentmask)) return True def gettxidx(self, txhash): ser_txhash = ser_uint256(txhash) try: ser_value = self.db.Get('tx:' + ser_txhash) except KeyError: return None pos = string.find(ser_value, ' ') txidx = TxIdx() txidx.blkhash = long(ser_value[:pos], 16) txidx.spentmask = long(ser_value[pos + 1:], 16) return txidx def gettx(self, txhash): txidx = self.gettxidx(txhash) if txidx is None: return None block = self.getblock(txidx.blkhash) if block: for tx in block.vtx: tx.calc_sha256() if tx.sha256 == txhash: return tx self.log.write("ERROR: Missing TX %064x in block %064x" % (txhash, txidx.blkhash)) return None def haveblock(self, blkhash, checkorphans): if self.blk_cache.exists(blkhash): return True if checkorphans and blkhash in self.orphans: return True ser_hash = ser_uint256(blkhash) try: self.db.Get('blocks:' + ser_hash) return True except KeyError: return False def have_prevblock(self, block): if self.getheight() < 0 and block.sha256 == self.netmagic.block0: return True if self.haveblock(block.hashPrevBlock, False): return True return False def getblock(self, blkhash): block = self.blk_cache.get(blkhash) if block is not None: return block ser_hash = ser_uint256(blkhash) try: # Lookup the block index, seek in the file fpos = long(self.db.Get('blocks:' + ser_hash)) self.blk_read.seek(fpos) # read and decode "block" msg recvbuf = self.blk_read.read(4 + 4) if recvbuf[:4] == 'ZLIB': msg_len = int(recvbuf[4:8].encode('hex'), 16) recvbuf = self.blk_read.read(msg_len) f = cStringIO.StringIO(zlib.decompress(recvbuf)) msg = message_read(self.netmagic, f) else: self.blk_read.seek(fpos) msg = message_read(self.netmagic, self.blk_read) if msg is None: return None block = msg.block except KeyError: return None self.blk_cache.put(blkhash, block) return block def spend_txout(self, txhash, n_idx, batch=None): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask |= (1L << n_idx) self.puttxidx(txhash, txidx, batch) return True def clear_txout(self, txhash, n_idx, batch=None): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask &= ~(1L << n_idx) self.puttxidx(txhash, txidx, batch) return True def unique_outpts(self, block): outpts = {} txmap = {} for tx in block.vtx: if tx.is_coinbase: continue txmap[tx.sha256] = tx for txin in tx.vin: v = (txin.prevout.hash, txin.prevout.n) if v in outs: return None outpts[v] = False return (outpts, txmap) def txout_spent(self, txout): txidx = self.gettxidx(txout.hash) if txidx is None: return None if txout.n > 100000: # outpoint index sanity check return None if txidx.spentmask & (1L << txout.n): return True return False
class ChainDb(object): def __init__(self, settings, datadir, log, mempool, netmagic, readonly=False, fast_dbm=False): self.settings = settings self.log = log self.mempool = mempool self.readonly = readonly self.netmagic = netmagic self.fast_dbm = fast_dbm self.blk_cache = Cache(750) self.orphans = {} self.orphan_deps = {} # LevelDB to hold: # tx:* transaction outputs # misc:* state # height:* list of blocks at height h # blkmeta:* block metadata # blocks:* block seek point in stream self.blk_write = io.BufferedWriter(io.FileIO(datadir + '/blocks.dat','ab')) self.blk_read = io.BufferedReader(io.FileIO(datadir + '/blocks.dat','rb')) self.db = leveldb.LevelDB(datadir + '/leveldb') try: self.db.Get('misc:height') except KeyError: self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE") batch = leveldb.WriteBatch() batch.Put('misc:height', str(-1)) batch.Put('misc:msg_start', self.netmagic.msg_start) batch.Put('misc:tophash', ser_uint256(0L)) batch.Put('misc:total_work', hex(0L)) self.db.Write(batch) try: start = self.db.Get('misc:msg_start') if start != self.netmagic.msg_start: raise KeyError except KeyError: self.log.write("Database magic number mismatch. Data corruption or incorrect network?") raise RuntimeError def puttxidx(self, txhash, txidx, batch=None): ser_txhash = ser_uint256(txhash) try: self.db.Get('tx:'+ser_txhash) old_txidx = self.gettxidx(txhash) self.log.write("WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x" % (txhash, self.getheight(), old_txidx.blkhash, old_txidx.spentmask, txidx.blkhash)) except KeyError: pass batch = self.db if batch is not None else batch batch.Put('tx:'+ser_txhash, hex(txidx.blkhash) + ' ' + hex(txidx.spentmask)) return True def gettxidx(self, txhash): ser_txhash = ser_uint256(txhash) try: ser_value = self.db.Get('tx:'+ser_txhash) except KeyError: return None pos = string.find(ser_value, ' ') txidx = TxIdx() txidx.blkhash = long(ser_value[:pos], 16) txidx.spentmask = long(ser_value[pos+1:], 16) return txidx def gettx(self, txhash): txidx = self.gettxidx(txhash) if txidx is None: return None block = self.getblock(txidx.blkhash) for tx in block.vtx: tx.calc_sha256() if tx.sha256 == txhash: return tx self.log.write("ERROR: Missing TX %064x in block %064x" % (txhash, txidx.blkhash)) return None def haveblock(self, blkhash, checkorphans): if self.blk_cache.exists(blkhash): return True if checkorphans and blkhash in self.orphans: return True ser_hash = ser_uint256(blkhash) try: self.db.Get('blocks:'+ser_hash) return True except KeyError: return False def have_prevblock(self, block): if self.getheight() < 0 and block.sha256 == self.netmagic.block0: return True if self.haveblock(block.hashPrevBlock, False): return True return False def getblock(self, blkhash): block = self.blk_cache.get(blkhash) if block is not None: return block ser_hash = ser_uint256(blkhash) try: # Lookup the block index, seek in the file fpos = long(self.db.Get('blocks:'+ser_hash)) self.blk_read.seek(fpos) # read and decode "block" msg msg = message_read(self.netmagic, self.blk_read) if msg is None: return None block = msg.block except KeyError: return None self.blk_cache.put(blkhash, block) return block def spend_txout(self, txhash, n_idx, batch=None): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask |= (1L << n_idx) self.puttxidx(txhash, txidx, batch) return True def clear_txout(self, txhash, n_idx, batch=None): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask &= ~(1L << n_idx) self.puttxidx(txhash, txidx, batch) return True def unique_outpts(self, block): outpts = {} txmap = {} for tx in block.vtx: if tx.is_coinbase: continue txmap[tx.sha256] = tx for txin in tx.vin: v = (txin.prevout.hash, txin.prevout.n) if v in outs: return None outpts[v] = False return (outpts, txmap) def spent_outpts(self, block): # list of outpoints this block wants to spend l = self.unique_outpts(block) if l is None: return None outpts = l[0] txmap = l[1] spendlist = {} # pass 1: if outpoint in db, make sure it is unspent for k in outpts.iterkeys(): txidx = self.gettxidx(k[0]) if txidx is None: continue if k[1] > 100000: # outpoint index sanity check return None if txidx.spentmask & (1L << k[1]): return None outpts[k] = True # skip in pass 2 # pass 2: remaining outpoints must exist in this block for k, v in outpts.iteritems(): if v: continue if k[0] not in txmap: # validate txout hash return None tx = txmap[k[0]] # validate txout index (n) if k[1] >= len(tx.vout): return None # outpts[k] = True # not strictly necessary return outpts.keys()
class ChainDb(object): def __init__(self, settings, datadir, log, mempool, netmagic, readonly=False, fast_dbm=False): self.settings = settings self.log = log self.mempool = mempool self.readonly = readonly self.netmagic = netmagic self.fast_dbm = fast_dbm self.blk_cache = Cache(750) self.orphans = {} self.orphan_deps = {} # LevelDB to hold: # tx:* transaction outputs # misc:* state # height:* list of blocks at height h # blkmeta:* block metadata # blocks:* block seek point in stream self.blk_write = io.BufferedWriter( io.FileIO(datadir + '/blocks.dat', 'ab')) self.blk_read = io.BufferedReader( io.FileIO(datadir + '/blocks.dat', 'rb')) self.db = leveldb.LevelDB(datadir + '/leveldb') try: self.db.Get('misc:height') except KeyError: self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE") batch = leveldb.WriteBatch() batch.Put('misc:height', str(-1)) batch.Put('misc:msg_start', self.netmagic.msg_start) batch.Put('misc:tophash', ser_uint256(0L)) batch.Put('misc:total_work', hex(0L)) self.db.Write(batch) try: start = self.db.Get('misc:msg_start') if start != self.netmagic.msg_start: raise KeyError except KeyError: self.log.write( "Database magic number mismatch. Data corruption or incorrect network?" ) raise RuntimeError def puttxidx(self, txhash, txidx, batch=None): ser_txhash = ser_uint256(txhash) try: self.db.Get('tx:' + ser_txhash) old_txidx = self.gettxidx(txhash) self.log.write( "WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x" % (txhash, self.getheight(), old_txidx.blkhash, old_txidx.spentmask, txidx.blkhash)) except KeyError: pass batch = self.db if batch is not None else batch batch.Put('tx:' + ser_txhash, hex(txidx.blkhash) + ' ' + hex(txidx.spentmask)) return True def gettxidx(self, txhash): ser_txhash = ser_uint256(txhash) try: ser_value = self.db.Get('tx:' + ser_txhash) except KeyError: return None pos = string.find(ser_value, ' ') txidx = TxIdx() txidx.blkhash = long(ser_value[:pos], 16) txidx.spentmask = long(ser_value[pos + 1:], 16) return txidx def gettx(self, txhash): txidx = self.gettxidx(txhash) if txidx is None: return None block = self.getblock(txidx.blkhash) for tx in block.vtx: tx.calc_sha256() if tx.sha256 == txhash: return tx self.log.write("ERROR: Missing TX %064x in block %064x" % (txhash, txidx.blkhash)) return None def haveblock(self, blkhash, checkorphans): if self.blk_cache.exists(blkhash): return True if checkorphans and blkhash in self.orphans: return True ser_hash = ser_uint256(blkhash) try: self.db.Get('blocks:' + ser_hash) return True except KeyError: return False def have_prevblock(self, block): if self.getheight() < 0 and block.sha256 == self.netmagic.block0: return True if self.haveblock(block.hashPrevBlock, False): return True return False def getblock(self, blkhash): block = self.blk_cache.get(blkhash) if block is not None: return block ser_hash = ser_uint256(blkhash) try: # Lookup the block index, seek in the file fpos = long(self.db.Get('blocks:' + ser_hash)) self.blk_read.seek(fpos) # read and decode "block" msg msg = message_read(self.netmagic, self.blk_read) if msg is None: return None block = msg.block except KeyError: return None self.blk_cache.put(blkhash, block) return block def spend_txout(self, txhash, n_idx, batch=None): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask |= (1L << n_idx) self.puttxidx(txhash, txidx, batch) return True def clear_txout(self, txhash, n_idx, batch=None): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask &= ~(1L << n_idx) self.puttxidx(txhash, txidx, batch) return True def unique_outpts(self, block): outpts = {} txmap = {} for tx in block.vtx: if tx.is_coinbase: continue txmap[tx.sha256] = tx for txin in tx.vin: v = (txin.prevout.hash, txin.prevout.n) if v in outs: return None outpts[v] = False return (outpts, txmap) def spent_outpts(self, block): # list of outpoints this block wants to spend l = self.unique_outpts(block) if l is None: return None outpts = l[0] txmap = l[1] spendlist = {} # pass 1: if outpoint in db, make sure it is unspent for k in outpts.iterkeys(): txidx = self.gettxidx(k[0]) if txidx is None: continue if k[1] > 100000: # outpoint index sanity check return None if txidx.spentmask & (1L << k[1]): return None outpts[k] = True # skip in pass 2 # pass 2: remaining outpoints must exist in this block for k, v in outpts.iteritems(): if v: continue if k[0] not in txmap: # validate txout hash return None tx = txmap[k[0]] # validate txout index (n) if k[1] >= len(tx.vout): return None # outpts[k] = True # not strictly necessary return outpts.keys()
class ChainDb(object): def __init__(self, settings, datadir, log, mempool, netmagic, readonly=False, fast_dbm=False,compression=False): self.settings = settings self.log = log self.mempool = mempool self.readonly = readonly self.netmagic = netmagic self.fast_dbm = fast_dbm self.blk_cache = Cache(1000) self.orphans = {} self.orphan_deps = {} self.compress_on_write = compression # LevelDB to hold: # tx:* transaction outputs # misc:* state # height:* list of blocks at height h # blkmeta:* block metadata # blocks:* block seek point in stream self.blk_write = io.BufferedWriter(io.FileIO(datadir + '/blocks.dat','ab')) self.blk_read = io.BufferedReader(io.FileIO(datadir + '/blocks.dat','rb')) self.db = leveldb.LevelDB(datadir + '/leveldb') try: self.db.Get('misc:height') except KeyError: self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE") batch = leveldb.WriteBatch() batch.Put('misc:height', str(-1)) batch.Put('misc:msg_start', self.netmagic.msg_start) batch.Put('misc:tophash', ser_uint256(0L)) batch.Put('misc:total_work', hex(0L)) self.db.Write(batch) try: start = self.db.Get('misc:msg_start') if start != self.netmagic.msg_start: raise KeyError except KeyError: self.log.write("Database magic number mismatch. Data corruption or incorrect network?") raise RuntimeError def puttxidx(self, txhash, txidx, batch=None): ser_txhash = ser_uint256(txhash) try: self.db.Get('tx:'+ser_txhash) old_txidx = self.gettxidx(txhash) self.log.write("WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x" % (txhash, self.getheight(), old_txidx.blkhash, old_txidx.spentmask, txidx.blkhash)) except KeyError: pass batch = self.db if batch is not None else batch batch.Put('tx:'+ser_txhash, hex(txidx.blkhash) + ' ' + hex(txidx.spentmask)) return True def gettxidx(self, txhash): ser_txhash = ser_uint256(txhash) try: ser_value = self.db.Get('tx:'+ser_txhash) except KeyError: return None pos = string.find(ser_value, ' ') txidx = TxIdx() txidx.blkhash = long(ser_value[:pos], 16) txidx.spentmask = long(ser_value[pos+1:], 16) return txidx def gettx(self, txhash): txidx = self.gettxidx(txhash) if txidx is None: return None block = self.getblock(txidx.blkhash) if block: for tx in block.vtx: tx.calc_sha256() if tx.sha256 == txhash: return tx self.log.write("ERROR: Missing TX %064x in block %064x" % (txhash, txidx.blkhash)) return None def haveblock(self, blkhash, checkorphans): if self.blk_cache.exists(blkhash): return True if checkorphans and blkhash in self.orphans: return True ser_hash = ser_uint256(blkhash) try: self.db.Get('blocks:'+ser_hash) return True except KeyError: return False def have_prevblock(self, block): if self.getheight() < 0 and block.sha256 == self.netmagic.block0: return True if self.haveblock(block.hashPrevBlock, False): return True return False def getblock(self, blkhash): block = self.blk_cache.get(blkhash) if block is not None: return block ser_hash = ser_uint256(blkhash) try: # Lookup the block index, seek in the file fpos = long(self.db.Get('blocks:'+ser_hash)) self.blk_read.seek(fpos) # read and decode "block" msg recvbuf = self.blk_read.read(4+4) if recvbuf[:4] == 'ZLIB': msg_len = int(recvbuf[4:8].encode('hex'),16) recvbuf = self.blk_read.read(msg_len) f = cStringIO.StringIO(zlib.decompress(recvbuf)) msg = message_read(self.netmagic, f) else: self.blk_read.seek(fpos) msg = message_read(self.netmagic, self.blk_read) if msg is None: return None block = msg.block except KeyError: return None self.blk_cache.put(blkhash, block) return block def spend_txout(self, txhash, n_idx, batch=None): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask |= (1L << n_idx) self.puttxidx(txhash, txidx, batch) return True def clear_txout(self, txhash, n_idx, batch=None): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask &= ~(1L << n_idx) self.puttxidx(txhash, txidx, batch) return True def unique_outpts(self, block): outpts = {} txmap = {} for tx in block.vtx: if tx.is_coinbase: continue txmap[tx.sha256] = tx for txin in tx.vin: v = (txin.prevout.hash, txin.prevout.n) if v in outs: return None outpts[v] = False return (outpts, txmap) def txout_spent(self, txout): txidx = self.gettxidx(txout.hash) if txidx is None: return None if txout.n > 100000: # outpoint index sanity check return None if txidx.spentmask & (1L << txout.n): return True return False
class ChainDb(object): def __init__(self, settings, datadir, log, mempool, netmagic, readonly=False, fast_dbm=False): self.settings = settings self.log = log self.mempool = mempool self.readonly = readonly self.netmagic = netmagic self.fast_dbm = fast_dbm self.blk_cache = Cache(750) self.orphans = {} self.orphan_deps = {} if readonly: mode_str = 'r' else: mode_str = 'c' if fast_dbm: self.log.write("Opening database in fast mode") mode_str += 'f' self.misc = gdbm.open(datadir + '/misc.dat', mode_str) self.blocks = gdbm.open(datadir + '/blocks.dat', mode_str) self.height = gdbm.open(datadir + '/height.dat', mode_str) self.blkmeta = gdbm.open(datadir + '/blkmeta.dat', mode_str) self.tx = gdbm.open(datadir + '/tx.dat', mode_str) if 'height' not in self.misc: self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE") self.misc['height'] = str(-1) self.misc['msg_start'] = self.netmagic.msg_start self.misc['tophash'] = ser_uint256(0L) self.misc['total_work'] = hex(0L) if 'msg_start' not in self.misc or (self.misc['msg_start'] != self.netmagic.msg_start): self.log.write("Database magic number mismatch. Data corruption or incorrect network?") raise RuntimeError def dbsync(self): self.misc.sync() self.blocks.sync() self.height.sync() self.blkmeta.sync() self.tx.sync() def puttxidx(self, txhash, txidx): ser_txhash = ser_uint256(txhash) if ser_txhash in self.tx: old_txidx = self.gettxidx(txhash) self.log.write("WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x" % (txhash, self.getheight(), old_txidx.blkhash, old_txidx.spentmask, txidx.blkhash)) self.tx[ser_txhash] = (hex(txidx.blkhash) + ' ' + hex(txidx.spentmask)) return True def gettxidx(self, txhash): ser_txhash = ser_uint256(txhash) if ser_txhash not in self.tx: return None ser_value = self.tx[ser_txhash] pos = string.find(ser_value, ' ') txidx = TxIdx() txidx.blkhash = long(ser_value[:pos], 16) txidx.spentmask = long(ser_value[pos+1:], 16) return txidx def gettx(self, txhash): txidx = self.gettxidx(txhash) if txidx is None: return None block = self.getblock(txidx.blkhash) for tx in block.vtx: tx.calc_sha256() if tx.sha256 == txhash: return tx self.log.write("ERROR: Missing TX %064x in block %064x" % (txhash, txidx.blkhash)) return None def haveblock(self, blkhash, checkorphans): if self.blk_cache.exists(blkhash): return True if checkorphans and blkhash in self.orphans: return True ser_hash = ser_uint256(blkhash) if ser_hash in self.blocks: return True return False def have_prevblock(self, block): if self.getheight() < 0 and block.sha256 == self.netmagic.block0: return True if self.haveblock(block.hashPrevBlock, False): return True return False def getblock(self, blkhash): block = self.blk_cache.get(blkhash) if block is not None: return block ser_hash = ser_uint256(blkhash) if ser_hash not in self.blocks: return None f = cStringIO.StringIO(self.blocks[ser_hash]) block = CBlock() block.deserialize(f) self.blk_cache.put(blkhash, block) return block def spend_txout(self, txhash, n_idx): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask |= (1L << n_idx) self.puttxidx(txhash, txidx) return True def clear_txout(self, txhash, n_idx): txidx = self.gettxidx(txhash) if txidx is None: return False txidx.spentmask &= ~(1L << n_idx) self.puttxidx(txhash, txidx) return True def unique_outpts(self, block): outpts = {} txmap = {} for tx in block.vtx: if tx.is_coinbase: continue txmap[tx.sha256] = tx for txin in tx.vin: v = (txin.prevout.hash, txin.prevout.n) if v in outs: return None outpts[v] = False return (outpts, txmap) def spent_outpts(self, block): # list of outpoints this block wants to spend l = self.unique_outpts(block) if l is None: return None outpts = l[0] txmap = l[1] spendlist = {} # pass 1: if outpoint in db, make sure it is unspent for k in outpts.iterkeys(): txidx = self.gettxidx(k[0]) if txidx is None: continue if k[1] > 100000: # outpoint index sanity check return None if txidx.spentmask & (1L << k[1]): return None outpts[k] = True # skip in pass 2 # pass 2: remaining outpoints must exist in this block for k, v in outpts.iteritems(): if v: continue if k[0] not in txmap: # validate txout hash return None tx = txmap[k[0]] # validate txout index (n) if k[1] >= len(tx.vout): return None # outpts[k] = True # not strictly necessary return outpts.keys()