def test_A(self):
tmpdbfile = os.path.join(self.datadir, '_tmp.db')
bref = MemBlock.read(333931)
proxy.set_rawmempool(333931)
proxy.blockcount = 333930
proxy.on = False
mempool = TxMempool(dbfile=tmpdbfile)
print("*** Proxy is OFF ***")
with mempool.context_start():
sleep(50)
proxy.on = True
print("*** Proxy is ON ***")
sleep(20)
proxy.blockcount = 333931
sleep(10)
btest = MemBlock.read(333931, dbfile=tmpdbfile)
# They're not equal because their times don't match.
self.assertNotEqual(btest, bref)
btest.time = bref.time
for entry in bref.entries.values():
entry.leadtime = int(entry.leadtime)
for entry in btest.entries.values():
entry.leadtime = btest.time - entry.time
self.assertEqual(btest, bref)
def test_writereadempty(self):
'''Tests write/read of empty entries dict'''
tmpdbfile = os.path.join(self.datadir, '_tmp.db')
memblock = MemBlock.read(self.test_blockheight)
memblock.entries = {}
memblock.write(dbfile=tmpdbfile, blocks_to_keep=2016)
memblock_read = MemBlock.read(self.test_blockheight,
dbfile=tmpdbfile)
self.assertEqual(memblock_read, memblock)
def test_writeread(self):
'''Tests that mempool entry is unchanged upon write/read.'''
tmpdbfile = os.path.join(self.datadir, '_tmp.db')
for height in MemBlock.get_heights():
memblock = MemBlock.read(height)
memblock.write(dbfile=tmpdbfile, blocks_to_keep=2016)
memblock_read = MemBlock.read(height, dbfile=tmpdbfile)
print(memblock_read)
self.assertEqual(memblock_read, memblock)
def test_deletehistory(self):
'''Test that history is deleted according to retention policy.'''
tmpdbfile = os.path.join(self.datadir, '_tmp.db')
blocks_to_keep = 10
memblocks = [MemBlock.read(height) for height in range(333931, 333954)]
for memblock in memblocks:
if memblock:
memblock.write(dbfile=tmpdbfile, blocks_to_keep=blocks_to_keep)
block_list = sorted(MemBlock.get_heights(dbfile=tmpdbfile))
self.assertEqual(len(block_list), blocks_to_keep)
self.assertEqual(block_list, list(range(333944, 333954)))
db = None
try:
db = sqlite3.connect(tmpdbfile)
blocktxsblocks = map(
itemgetter(0),
sorted(
db.execute("SELECT DISTINCT blockheight FROM blocktxs").
fetchall()))
self.assertEqual(block_list, blocktxsblocks)
txsheights = map(
itemgetter(0),
sorted(
db.execute(
"SELECT DISTINCT heightremoved FROM txs").fetchall()))
txsheights.remove(None)
self.assertEqual(block_list, txsheights)
# Check no duplicate txids.
self.assertEqual(
len(db.execute("SELECT DISTINCT txid FROM txs").fetchall()),
len(db.execute("SELECT txid FROM txs").fetchall()))
# Check the null-heightremoved txs
self.assertEqual(
db.execute("SELECT count(*) FROM txs "
"WHERE heightremoved IS NULL").fetchall()[0][0],
len(
filter(lambda entry: not entry.inblock,
memblocks[-1].entries.values())))
finally:
if db is not None:
db.close()
def setUp(self):
self.test_blockheight = 333931
self.memblockref = MemBlock.read(self.test_blockheight, dbfile=dbfile)
for entry in self.memblockref.entries.values():
entry.isconflict = False
self.testrawmempool = rawmempool_from_mementries(
self.memblockref.entries)
self.mempool = TxMempool(dbfile=None)
def test_duplicate_writes(self):
tmpdbfile = os.path.join(self.datadir, '_tmp.db')
block = MemBlock.read(333931)
block.write(tmpdbfile, 100)
self.assertRaises(sqlite3.IntegrityError, block.write, tmpdbfile, 100)
db = None
try:
db = sqlite3.connect(tmpdbfile)
txids = db.execute(
'SELECT txid FROM txs JOIN blocktxs '
'ON txs.id=blocktxs.txrowid WHERE blockheight=333931')
txids = [e[0] for e in txids]
self.assertEqual(sorted(set(txids)), sorted(txids))
block_read = MemBlock.read(333931, dbfile=tmpdbfile)
self.assertEqual(block, block_read)
finally:
if db is not None:
db.close()
def setUp(self):
self.test_blockheight = 333931
self.memblockref = MemBlock.read(self.test_blockheight,
dbfile=dbfile)
for entry in self.memblockref.entries.values():
entry.isconflict = False
self.testrawmempool = rawmempool_from_mementries(
self.memblockref.entries)
self.mempool = TxMempool(dbfile=None)
def get_mytime():
starttime = time()
b = MemBlock.read(333952, dbfile=memblock_dbfile)
reftime = b.time
def mytime():
return time() - starttime + reftime
return mytime
def test_B(self):
# Test the setting of rawmempool
proxy.set_rawmempool(333931)
rawmempool = proxy.getrawmempool()
b = MemBlock.read(333931, dbfile=dbfile)
self.assertEqual(set(b.entries), set(rawmempool))
for txid, rawentry in rawmempool.items():
for key, val in rawentry.items():
self.assertEqual(val, getattr(b.entries[txid], key))
def test_deletehistory(self):
'''Test that history is deleted according to retention policy.'''
tmpdbfile = os.path.join(self.datadir, '_tmp.db')
blocks_to_keep = 10
memblocks = [MemBlock.read(height)
for height in range(333931, 333954)]
for memblock in memblocks:
if memblock:
memblock.write(dbfile=tmpdbfile,
blocks_to_keep=blocks_to_keep)
block_list = sorted(MemBlock.get_heights(dbfile=tmpdbfile))
self.assertEqual(len(block_list), blocks_to_keep)
self.assertEqual(block_list, list(range(333944, 333954)))
db = None
try:
db = sqlite3.connect(tmpdbfile)
blocktxsblocks = map(itemgetter(0), sorted(db.execute(
"SELECT DISTINCT blockheight FROM blocktxs").fetchall()))
self.assertEqual(block_list, blocktxsblocks)
txsheights = map(itemgetter(0), sorted(db.execute(
"SELECT DISTINCT heightremoved FROM txs").fetchall()))
txsheights.remove(None)
self.assertEqual(block_list, txsheights)
# Check no duplicate txids.
self.assertEqual(
len(db.execute("SELECT DISTINCT txid FROM txs").fetchall()),
len(db.execute("SELECT txid FROM txs").fetchall()))
# Check the null-heightremoved txs
self.assertEqual(
db.execute("SELECT count(*) FROM txs "
"WHERE heightremoved IS NULL").fetchall()[0][0],
len(filter(
lambda entry: not entry.inblock,
memblocks[-1].entries.values()))
)
finally:
if db is not None:
db.close()
def test_duplicate_writes(self):
tmpdbfile = os.path.join(self.datadir, '_tmp.db')
block = MemBlock.read(333931)
block.write(tmpdbfile, 100)
self.assertRaises(
sqlite3.IntegrityError, block.write, tmpdbfile, 100)
db = None
try:
db = sqlite3.connect(tmpdbfile)
txids = db.execute(
'SELECT txid FROM txs JOIN blocktxs '
'ON txs.id=blocktxs.txrowid WHERE blockheight=333931')
txids = [e[0] for e in txids]
self.assertEqual(sorted(set(txids)), sorted(txids))
block_read = MemBlock.read(333931, dbfile=tmpdbfile)
self.assertEqual(block, block_read)
finally:
if db is not None:
db.close()
def start(self, blockrangetuple, stopflag=None, dbfile=MEMBLOCK_DBFILE):
logger.info("Beginning NP pool estimation "
"from blockrange({}, {})".format(*blockrangetuple))
self._clear_window(blockrangetuple[0])
for height in range(*blockrangetuple):
if height in self.blockstats:
continue
if stopflag and stopflag.is_set():
raise StopIteration("Stop flag set.")
memblock = MemBlock.read(height, dbfile=dbfile)
if memblock is None:
continue
self.update(memblock, is_init=True)
self._calc_estimates()
logger.info("Finished NP pool estimation.")
def test_A(self):
for height in range(333931, 333954):
b = MemBlock.read(height, dbfile=dbfile)
if b is not None:
min_leadtime = _calc_min_leadtime(b)
print("Block {}: the min leadtime is {}.".
format(height, min_leadtime))
txs = tx_preprocess(b)
for entry in b.entries.values():
if (entry.feerate, entry.inblock) not in txs:
self.assertTrue(
entry.is_high_priority() or
entry.leadtime < min_leadtime or
_depcheck(entry, b.entries) or
entry.isconflict
)
def start(self, blockrangetuple, stopflag=None, dbfile=MEMBLOCK_DBFILE):
logger.info("Starting TxRate estimation "
"from blockrange ({}, {}).".format(*blockrangetuple))
starttime = time()
self._reset_params()
prevblock = None
for height in range(*blockrangetuple):
if stopflag and stopflag.is_set():
raise StopIteration("Stop flag set.")
block = MemBlock.read(height, dbfile=dbfile)
self._addblock(block, prevblock)
prevblock = block
if self.totaltxs < 0 or self.totaltime <= 0:
raise ValueError("Insufficient number of blocks.")
self.txrate = self.totaltxs / self.totaltime
logger.info("Finished TxRate estimation in %.2f seconds." %
(time()-starttime))
def test_B(self):
"""Test estimation."""
with tmpdatadir_context():
pe = PoolsEstimatorNP()
pe.start((333931, 333954))
# A fake memblock with zero entries.
empty_memblock = MemBlock()
empty_memblock.blockheight = 333954
empty_memblock.height = 333953
empty_memblock.blocksize = 0
empty_memblock.time = pe.blockstats[333953][2] + 20
empty_memblock.entries = {}
pe.update(empty_memblock)
print(pe)
def populate_testdb(self):
t = 0
mempool = SimMempool({})
tx_emitter = txref.get_emitter(mempool)
print("txref is {}".format(txref))
for height in range(*self.gen_blockrange):
blockinterval = expovariate(1/600)
t += blockinterval
tx_emitter(blockinterval)
mempool_entries = mempool.get_entries()
entries = {}
for txid, entry in mempool_entries.items():
# Dummy fields
mementry = MemEntry()
mementry.startingpriority = 0
mementry.currentpriority = 0
mementry.fee = entry.feerate*entry.size
mementry.feerate = entry.feerate
mementry.leadtime = 0
mementry.isconflict = False
mementry.inblock = False
# Relevant fields
mementry.time = t - random()*blockinterval
mementry.height = height
entries[str(height)+txid] = mementry
mementry.size = entry.size
b = MemBlock()
b.height = height - 1
b.blockheight = height
b.time = t
b.blocksize = sum([
entry.size for entry in mempool_entries.values()])
b.entries = entries
b.write(self.tmpdbfile, 2000)
mempool.reset()
def start(self, blockheight, stopflag=None, dbfile=MEMBLOCK_DBFILE):
self._reset_params()
starttime = time()
num_blocks_to_use = int(log(0.01) / log(self._alpha) / 600)
startblock = blockheight - num_blocks_to_use + 1
blockrangetuple = (startblock, blockheight+1)
logger.info("Starting TxRate estimation "
"from blockrange ({}, {}).".format(*blockrangetuple))
for height in range(*blockrangetuple):
if stopflag and stopflag.is_set():
raise StopIteration("Stop flag set.")
block = MemBlock.read(height, dbfile=dbfile)
self.update(block, is_init=True)
self._calc_txrate()
logger.info("Finished TxRate estimation in %.2f seconds." %
(time()-starttime))
def waitmeasure(startheight, endheight, dbfile=MEMBLOCK_DBFILE):
blacklist = set()
txs = []
for height in range(startheight, endheight + 1):
block = MemBlock.read(height, dbfile=dbfile)
if block is None:
continue
# Don't consider txs with high priority or those with mempool deps.
blacklist.update(
set([
txid for txid, entry in block.entries.items()
if entry.is_high_priority() or entry.depends
]))
txs.extend([(entry.feerate, block.time - entry.time)
for txid, entry in block.entries.items()
if txid not in blacklist and entry.inblock])
blacklist = blacklist & set(block.entries)
return txs
def estimate(self, windowlen, stopflag=None, dbfile=MEMBLOCK_DBFILE):
totalhashes = sum([block.hashes for block in self.blocks])
self.hashrate = totalhashes / windowlen
self.maxblocksize = max(map(attrgetter("size"), self.blocks))
txs = []
feelimitedblocks = []
sizelimitedblocks = []
for blockmeta in self.blocks:
# We assume a block is fee-limited if its size is more than 10 kB
# smaller than the max block size.
# TODO: find a better way of choosing the margin size.
if self.maxblocksize - blockmeta.size > 10000:
feelimitedblocks.append(blockmeta)
else:
sizelimitedblocks.append(blockmeta)
if feelimitedblocks:
# For minfeerate estimation, prioritize medium-sized
# and recent blocks.
# We should avoid both small blocks (where minblocksize
# and blockprioritysize effects may be in play) and large blocks
# (max block size may have been reached). Separating
# feelimitedblocks and sizelimitedblocks works most of the time,
# however when pools are changing their max block size policy,
# it would lead to inaccurate results.
# Prioritizing recent blocks helps in the case where pools are
# changing their minfeerate policy.
meanblocksize = (
sum(map(attrgetter("size"), feelimitedblocks)) /
len(feelimitedblocks))
blockscores = [[block, 0] for block in feelimitedblocks]
blockscores.sort(key=lambda b: abs(b[0].size-meanblocksize))
for idx, blockscore in enumerate(blockscores):
blockscore[1] = max(idx, blockscore[1])
blockscores.sort(key=lambda b: b[0].height, reverse=True)
for idx, blockscore in enumerate(blockscores):
blockscore[1] = max(idx, blockscore[1])
blockscores.sort(key=itemgetter(1))
feelimitedblocks, dummy = zip(*blockscores)
nummissingblocks = 0
for blockmeta in feelimitedblocks:
if stopflag and stopflag.is_set():
raise StopIteration("Stop flag set.")
memblock = MemBlock.read(blockmeta.height, dbfile=dbfile)
if memblock is None:
nummissingblocks += 1
continue
txs.extend(tx_preprocess(memblock))
# Only take up to MAX_TXS txs.
# The optimal figure will depend on the tx byte rate profile:
# are there sufficient transactions with a feerate close to
# the pool's minfeerate? In the future MAX_TXS could be selected
# automatically.
if len(txs) >= MAX_TXS:
break
if not txs and sizelimitedblocks:
# All the blocks are close to the max block size.
# This should happen rarely, so we just choose the smallest block.
smallestblock = min(sizelimitedblocks, key=attrgetter("size"))
memblock = MemBlock.read(smallestblock.height, dbfile=dbfile)
if memblock:
txs.extend(tx_preprocess(memblock))
if txs:
self.mfrstats = calc_stranding_feerate(txs)
self.minfeerate = self.mfrstats['sfr']
else:
logger.warning("Pool estimation: no valid transactions.")
self.mfrstats = {
"sfr": float("inf"),
"bias": float("inf"),
"mean": float("inf"),
"std": float("inf"),
"abovekn": (-1, -1),
"belowkn": (-1, -1),
}
if nummissingblocks:
logger.warning("MFR estimation: {} missing blocks.".
format(nummissingblocks))
def set_rawmempool(self, height):
'''Set the rawmempool from test memblock with specified height.'''
b = MemBlock.read(height, dbfile=dbfile)
self.rawmempool = rawmempool_from_mementries(b.entries)
import cProfile
from feemodel.txmempool import MemBlock
from feemodel.simul.transient import transientsim
from feemodel.simul import Simul
from feemodel.util import DataSample
from feemodel.tests.config import test_memblock_dbfile as dbfile, poolsref, txref
# flake8: noqa
print(poolsref)
init_entries = MemBlock.read(333931, dbfile=dbfile).entries
sim = Simul(poolsref, txref)
print("Starting transientsim.")
cProfile.run("feepoints, waittimes = transientsim("
"sim, init_entries=init_entries, numprocesses=1)")
print("Completed with {} iters.".format(len(waittimes[0])))
print("Feerate\tMean wait")
for feerate, waitsample in zip(feepoints, waittimes):
waitdata = DataSample(waitsample)
waitdata.calc_stats()
print("{}\t{}".format(feerate, waitdata.mean))
def set_rawmempool(self, height):
'''Set the rawmempool from test memblock with specified height.'''
b = MemBlock.read(height, dbfile=dbfile)
self.rawmempool = rawmempool_from_mementries(b.entries)
def test_write_uninitialized(self):
'''Test write of uninitialized MemBlock.'''
tmpdbfile = os.path.join(self.datadir, '_tmp.db')
memblock = MemBlock()
with self.assertRaises(ValueError):
memblock.write(dbfile=tmpdbfile, blocks_to_keep=2016)
def test_read_uninitialized(self):
'''Read from a db that has not been initialized.'''
block = MemBlock.read(333931, dbfile='nonsense.db')
self.assertIsNone(block)
heights = MemBlock.get_heights(dbfile='nonsense.db')
self.assertEqual([], heights)
def test_write_uninitialized(self):
'''Test write of uninitialized MemBlock.'''
tmpdbfile = os.path.join(self.datadir, '_tmp.db')
memblock = MemBlock()
with self.assertRaises(ValueError):
memblock.write(dbfile=tmpdbfile, blocks_to_keep=2016)
def test_read_uninitialized(self):
'''Read from a db that has not been initialized.'''
block = MemBlock.read(333931, dbfile='nonsense.db')
self.assertIsNone(block)
heights = MemBlock.get_heights(dbfile='nonsense.db')
self.assertEqual([], heights)