def reparse(testnet=True):
"""
Reparse all transaction from the database.
- Create a new in-memory DB, copy the DB that is on-disk
- Reparse DB, automatically compares consensus hashes to the original ones from the on-disk DB
"""
options = dict(COUNTERPARTYD_OPTIONS)
server.initialise(database_file=':memory:', testnet=testnet, **options)
logger = logging.getLogger()
if testnet:
config.PREFIX = b'TESTXXXX'
memory_db = database.get_connection(read_only=False)
data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True)
prod_db_path = os.path.join(data_dir, '{}{}.db'.format(config.APP_NAME, '.testnet' if testnet else ''))
assert os.path.exists(prod_db_path), "database path {} does not exist".format(prod_db_path)
prod_db = apsw.Connection(prod_db_path)
prod_db.setrowtrace(database.rowtracer)
# Copy DB from file on disk (should be a DB file with at least all the checkpoints)
# in-memory DB shouldn't have been written to yet up until this point
with memory_db.backup("main", prod_db, "main") as backup:
while not backup.done:
backup.step(100)
# Drop most tables (except blocks, transactions, undolog)
memory_cursor = memory_db.cursor()
for table in blocks.TABLES + ['balances']:
memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))
# Check that all checkpoint blocks are in the database to be tested.
if testnet:
CHECKPOINTS = check.CHECKPOINTS_TESTNET
else:
CHECKPOINTS = check.CHECKPOINTS_MAINNET
for block_index in CHECKPOINTS.keys():
block_exists = bool(list(memory_cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,))))
assert block_exists, "block #%d does not exist" % block_index
# Clean consensus hashes if first block hash don’t match with checkpoint.
checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
columns = [column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')]
for field in ['ledger_hash', 'txlist_hash']:
if field in columns:
sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(field)
first_hash = list(memory_cursor.execute(sql, (config.BLOCK_FIRST,)))[0][field]
if first_hash != checkpoints[config.BLOCK_FIRST][field]:
logger.info('First hash changed. Cleaning {}.'.format(field))
memory_cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field))
# Initialise missing tables
blocks.initialise(memory_db)
previous_ledger_hash = None
previous_txlist_hash = None
previous_messages_hash = None
# Reparse each block, if ConsensusError is thrown then the difference
memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
for block in memory_cursor.fetchall():
try:
util.CURRENT_BLOCK_INDEX = block['block_index']
previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block(
memory_db, block['block_index'], block['block_time'],
previous_ledger_hash=previous_ledger_hash, ledger_hash=block['ledger_hash'],
previous_txlist_hash=previous_txlist_hash, txlist_hash=block['txlist_hash'],
previous_messages_hash=previous_messages_hash)
logger.info('Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' % (
block['block_index'], previous_ledger_hash[-5:], previous_txlist_hash[-5:], previous_messages_hash[-5:],
(' [overwrote %s]' % previous_found_messages_hash) if previous_found_messages_hash and previous_found_messages_hash != previous_messages_hash else ''))
except check.ConsensusError as e:
message = str(e)
if message.find('ledger_hash') != -1:
new_ledger = get_block_ledger(memory_db, block['block_index'])
old_ledger = get_block_ledger(prod_db, block['block_index'])
compare_strings(old_ledger, new_ledger)
elif message.find('txlist_hash') != -1:
new_txlist = get_block_txlist(memory_db, block['block_index'])
old_txlist = get_block_txlist(prod_db, block['block_index'])
compare_strings(old_txlist, new_txlist)
raise e
def reparse(testnet=True):
"""Reparse all transaction from the database, create a new blockchain and compare it to the old one."""
options = dict(COUNTERPARTYD_OPTIONS)
server.initialise(database_file=':memory:', testnet=testnet, **options)
if testnet:
config.PREFIX = b'TESTXXXX'
logger = logging.getLogger()
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logger.addHandler(console)
memory_db = database.get_connection(read_only=False)
initialise_db(memory_db)
data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True)
prod_db_path = os.path.join(data_dir, '{}{}.db'.format(config.APP_NAME, '.testnet' if testnet else ''))
prod_db = apsw.Connection(prod_db_path)
prod_db.setrowtrace(database.rowtracer)
with memory_db.backup("main", prod_db, "main") as backup:
backup.step()
# Here we don’t use block.reparse() because it reparse db in transaction (`with db`).
memory_cursor = memory_db.cursor()
for table in blocks.TABLES + ['balances']:
memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))
# Check that all checkpoint blocks are in the database to be tested.
if testnet:
CHECKPOINTS = check.CHECKPOINTS_TESTNET
else:
CHECKPOINTS = check.CHECKPOINTS_MAINNET
for block_index in CHECKPOINTS.keys():
block_exists = bool(list(memory_cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,))))
assert block_exists
# Clean consensus hashes if first block hash don’t match with checkpoint.
checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
columns = [column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')]
for field in ['ledger_hash', 'txlist_hash']:
if field in columns:
sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(field)
first_hash = list(memory_cursor.execute(sql, (config.BLOCK_FIRST,)))[0][field]
if first_hash != checkpoints[config.BLOCK_FIRST][field]:
logger.info('First hash changed. Cleaning {}.'.format(field))
memory_cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field))
blocks.initialise(memory_db)
previous_ledger_hash = None
previous_txlist_hash = None
previous_messages_hash = None
memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
for block in memory_cursor.fetchall():
try:
util.CURRENT_BLOCK_INDEX = block['block_index']
previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block(
memory_db, block['block_index'], block['block_time'],
previous_ledger_hash=previous_ledger_hash, ledger_hash=block['ledger_hash'],
previous_txlist_hash=previous_txlist_hash, txlist_hash=block['txlist_hash'],
previous_messages_hash=previous_messages_hash)
logger.info('Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' % (
block['block_index'], previous_ledger_hash[-5:], previous_txlist_hash[-5:], previous_messages_hash[-5:],
(' [overwrote %s]' % previous_found_messages_hash) if previous_found_messages_hash and previous_found_messages_hash != previous_messages_hash else ''))
except check.ConsensusError as e:
message = str(e)
if message.find('ledger_hash') != -1:
new_ledger = get_block_ledger(memory_db, block['block_index'])
old_ledger = get_block_ledger(prod_db, block['block_index'])
compare_strings(old_ledger, new_ledger)
elif message.find('txlist_hash') != -1:
new_txlist = get_block_txlist(memory_db, block['block_index'])
old_txlist = get_block_txlist(prod_db, block['block_index'])
compare_strings(old_txlist, new_txlist)
raise(e)
def initialise_db(db):
"""Initialise blockchain in the db and insert first block."""
blocks.initialise(db)
insert_block(db, config.BLOCK_FIRST - 1, parse_block=True)
def initialise_db(db):
"""Initialise blockchain in the db and insert first block."""
blocks.initialise(db)
insert_block(db, config.BURN_START - 1)
def reparse(testnet=True):
"""Reparse all transaction from the database, create a new blockchain and compare it to the old one."""
options = dict(COUNTERPARTYD_OPTIONS)
server.initialise(database_file=':memory:', testnet=testnet, **options)
logger = logging.getLogger()
if testnet:
config.PREFIX = b'TESTXXXX'
memory_db = database.get_connection(read_only=False)
initialise_db(memory_db)
data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME,
appname=config.APP_NAME,
roaming=True)
prod_db_path = os.path.join(
data_dir, '{}{}.db'.format(config.APP_NAME,
'.testnet' if testnet else ''))
assert os.path.exists(
prod_db_path), "database path {} does not exist".format(prod_db_path)
prod_db = apsw.Connection(prod_db_path)
prod_db.setrowtrace(database.rowtracer)
with memory_db.backup("main", prod_db, "main") as backup:
backup.step()
# Here we don’t use block.reparse() because it reparse db in transaction (`with db`).
memory_cursor = memory_db.cursor()
for table in blocks.TABLES + ['balances']:
memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))
# Check that all checkpoint blocks are in the database to be tested.
if testnet:
CHECKPOINTS = check.CHECKPOINTS_TESTNET
else:
CHECKPOINTS = check.CHECKPOINTS_MAINNET
for block_index in CHECKPOINTS.keys():
block_exists = bool(
list(
memory_cursor.execute(
'''SELECT * FROM blocks WHERE block_index = ?''',
(block_index, ))))
assert block_exists, "block #%d does not exist" % block_index
# Clean consensus hashes if first block hash don’t match with checkpoint.
checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
columns = [
column['name']
for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')
]
for field in ['ledger_hash', 'txlist_hash']:
if field in columns:
sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(
field)
first_hash = list(
memory_cursor.execute(sql, (config.BLOCK_FIRST, )))[0][field]
if first_hash != checkpoints[config.BLOCK_FIRST][field]:
logger.info('First hash changed. Cleaning {}.'.format(field))
memory_cursor.execute(
'''UPDATE blocks SET {} = NULL'''.format(field))
blocks.initialise(memory_db)
previous_ledger_hash = None
previous_txlist_hash = None
previous_messages_hash = None
memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
for block in memory_cursor.fetchall():
try:
util.CURRENT_BLOCK_INDEX = block['block_index']
previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block(
memory_db,
block['block_index'],
block['block_time'],
previous_ledger_hash=previous_ledger_hash,
ledger_hash=block['ledger_hash'],
previous_txlist_hash=previous_txlist_hash,
txlist_hash=block['txlist_hash'],
previous_messages_hash=previous_messages_hash)
logger.info(
'Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' %
(block['block_index'], previous_ledger_hash[-5:],
previous_txlist_hash[-5:], previous_messages_hash[-5:],
(' [overwrote %s]' %
previous_found_messages_hash) if previous_found_messages_hash
and previous_found_messages_hash != previous_messages_hash
else ''))
except check.ConsensusError as e:
message = str(e)
if message.find('ledger_hash') != -1:
new_ledger = get_block_ledger(memory_db, block['block_index'])
old_ledger = get_block_ledger(prod_db, block['block_index'])
compare_strings(old_ledger, new_ledger)
elif message.find('txlist_hash') != -1:
new_txlist = get_block_txlist(memory_db, block['block_index'])
old_txlist = get_block_txlist(prod_db, block['block_index'])
compare_strings(old_txlist, new_txlist)
raise (e)
def initialise_db(db):
"""Initialise blockchain in the db and insert first block."""
blocks.initialise(db)
insert_block(db, config.BLOCK_FIRST - 1, parse_block=True)
def reparse(testnet=True):
"""
Reparse all transaction from the database.
- Create a new in-memory DB, copy the DB that is on-disk
- Reparse DB, automatically compares consensus hashes to the original ones from the on-disk DB
"""
options = dict(COUNTERPARTYD_OPTIONS)
server.initialise(database_file=':memory:', testnet=testnet, **options)
logger = logging.getLogger()
if testnet:
config.PREFIX = b'TESTXXXX'
memory_db = database.get_connection(read_only=False)
data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME,
appname=config.APP_NAME,
roaming=True)
prod_db_path = os.path.join(
data_dir, '{}{}.db'.format(config.APP_NAME,
'.testnet' if testnet else ''))
assert os.path.exists(
prod_db_path), "database path {} does not exist".format(prod_db_path)
prod_db = apsw.Connection(prod_db_path)
prod_db.setrowtrace(database.rowtracer)
# Copy DB from file on disk (should be a DB file with at least all the checkpoints)
# in-memory DB shouldn't have been written to yet up until this point
with memory_db.backup("main", prod_db, "main") as backup:
while not backup.done:
backup.step(100)
# Drop most tables (except blocks, transactions, undolog)
memory_cursor = memory_db.cursor()
for table in blocks.TABLES + ['balances']:
memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))
# Check that all checkpoint blocks are in the database to be tested.
if testnet:
CHECKPOINTS = check.CHECKPOINTS_TESTNET
else:
CHECKPOINTS = check.CHECKPOINTS_MAINNET
for block_index in CHECKPOINTS.keys():
block_exists = bool(
list(
memory_cursor.execute(
'''SELECT * FROM blocks WHERE block_index = ?''',
(block_index, ))))
assert block_exists, "block #%d does not exist" % block_index
# Clean consensus hashes if first block hash don’t match with checkpoint.
checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
columns = [
column['name']
for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')
]
for field in ['ledger_hash', 'txlist_hash']:
if field in columns:
sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(
field)
first_hash = list(
memory_cursor.execute(sql, (config.BLOCK_FIRST, )))[0][field]
if first_hash != checkpoints[config.BLOCK_FIRST][field]:
logger.info('First hash changed. Cleaning {}.'.format(field))
memory_cursor.execute(
'''UPDATE blocks SET {} = NULL'''.format(field))
# Initialise missing tables
blocks.initialise(memory_db)
previous_ledger_hash = None
previous_txlist_hash = None
previous_messages_hash = None
# Reparse each block, if ConsensusError is thrown then the difference
memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
for block in memory_cursor.fetchall():
try:
util.CURRENT_BLOCK_INDEX = block['block_index']
previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block(
memory_db,
block['block_index'],
block['block_time'],
previous_ledger_hash=previous_ledger_hash,
ledger_hash=block['ledger_hash'],
previous_txlist_hash=previous_txlist_hash,
txlist_hash=block['txlist_hash'],
previous_messages_hash=previous_messages_hash)
logger.info(
'Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' %
(block['block_index'], previous_ledger_hash[-5:],
previous_txlist_hash[-5:], previous_messages_hash[-5:],
(' [overwrote %s]' %
previous_found_messages_hash) if previous_found_messages_hash
and previous_found_messages_hash != previous_messages_hash
else ''))
except check.ConsensusError as e:
message = str(e)
if message.find('ledger_hash') != -1:
new_ledger = get_block_ledger(memory_db, block['block_index'])
old_ledger = get_block_ledger(prod_db, block['block_index'])
compare_strings(old_ledger, new_ledger)
elif message.find('txlist_hash') != -1:
new_txlist = get_block_txlist(memory_db, block['block_index'])
old_txlist = get_block_txlist(prod_db, block['block_index'])
compare_strings(old_txlist, new_txlist)
raise e
def initialise_db(db):
"""Initialise blockchain in the db and insert first block."""
blocks.initialise(db)
insert_block(db, config.BURN_START - 1)
