def do_book(testnet):
# Filenames.
if testnet:
filename = 'book.testnet'
else:
filename = 'book.mainnet'
old = CURR_DIR + '/' + filename
new = old + '.new'
# Get last block_index of old book.
with open(old, 'r') as f:
block_index = int(f.readlines()[-1][7:13])
# Use temporary DB.
counterpartyd.set_options(testnet=testnet, carefulness=60)
default_db = config.DATABASE
temp_db = tempfile.gettempdir() + '/' + os.path.basename(config.DATABASE)
shutil.copyfile(default_db, temp_db)
counterpartyd.set_options(database_file=temp_db,
testnet=testnet,
carefulness=60)
db = util.connect_to_db()
cursor = db.cursor()
# TODO: USE API
import subprocess
if testnet:
subprocess.check_call([
'counterpartyd.py', '--database-file=' + temp_db, '--testnet',
'--force', '--carefulness=60', 'reparse'
])
else:
subprocess.check_call([
'counterpartyd.py', '--database-file=' + temp_db,
'--carefulness=60', 'reparse'
])
# Get new book.
with open(new, 'w') as f:
# Credits.
cursor.execute(
'select * from credits where block_index <= ? order by block_index, address, asset',
(block_index, ))
for credit in list(cursor):
f.write('credit ' + str(summarise(credit)) + '\n')
# Debits.
cursor.execute(
'select * from debits where block_index <= ? order by block_index, address, asset',
(block_index, ))
for debit in cursor.fetchall():
f.write('debit ' + str(summarise(debit)) + '\n')
# Compare books.
compare(filename)
# Clean up.
cursor.close()
os.remove(temp_db)
def initialise_rawtransactions_db(db):
if pytest.config.option.savescenarios:
counterpartyd.set_options(testnet=True, **COUNTERPARTYD_OPTIONS)
cursor = db.cursor()
cursor.execute('DROP TABLE IF EXISTS raw_transactions')
cursor.execute('CREATE TABLE IF NOT EXISTS raw_transactions(tx_hash TEXT UNIQUE, tx_hex TEXT, tx_json TEXT)')
with open(CURR_DIR + '/fixtures/unspent_outputs.json', 'r') as listunspent_test_file:
wallet_unspent = json.load(listunspent_test_file)
for output in wallet_unspent:
txid = binascii.hexlify(bitcoinlib.core.lx(output['txid'])).decode()
tx = bitcoin.decode_raw_transaction(output['txhex'])
cursor.execute('INSERT INTO raw_transactions VALUES (?, ?, ?)', (txid, output['txhex'], json.dumps(tx)))
cursor.close()
def initialise_rawtransactions_db(db):
if pytest.config.option.savescenarios:
counterpartyd.set_options(testnet=True, **COUNTERPARTYD_OPTIONS)
cursor = db.cursor()
cursor.execute('DROP TABLE IF EXISTS raw_transactions')
cursor.execute('CREATE TABLE IF NOT EXISTS raw_transactions(tx_hash TEXT UNIQUE, tx_hex TEXT)')
with open(CURR_DIR + '/fixtures/unspent_outputs.json', 'r') as listunspent_test_file:
wallet_unspent = json.load(listunspent_test_file)
for output in wallet_unspent:
txid = binascii.hexlify(bitcoinlib.core.lx(output['txid'])).decode()
tx = backend.deserialize(output['txhex'])
cursor.execute('INSERT INTO raw_transactions VALUES (?, ?)', (txid, output['txhex']))
cursor.close()
def reparse(testnet=True):
options = dict(COUNTERPARTYD_OPTIONS)
options.pop('data_dir')
counterpartyd.set_options(database_file=':memory:', testnet=testnet, **options)
if testnet:
config.PREFIX = b'TESTXXXX'
logger = logging.getLogger()
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logger.addHandler(console)
memory_db = util.connect_to_db()
initialise_db(memory_db)
prod_db_path = os.path.join(config.DATA_DIR, '{}.{}{}.db'.format(config.XCP_CLIENT, str(config.VERSION_MAJOR), '.testnet' if testnet else ''))
prod_db = apsw.Connection(prod_db_path)
prod_db.setrowtrace(util.rowtracer)
with memory_db.backup("main", prod_db, "main") as backup:
backup.step()
# here we don't use block.reparse() because it reparse db in transaction (`with db`)
memory_cursor = memory_db.cursor()
for table in blocks.TABLES + ['balances']:
memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))
blocks.initialise(memory_db)
previous_ledger_hash = None
previous_txlist_hash = None
memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
for block in memory_cursor.fetchall():
try:
logger.info('Block (re‐parse): {}'.format(str(block['block_index'])))
previous_ledger_hash, previous_txlist_hash = blocks.parse_block(memory_db, block['block_index'], block['block_time'],
previous_ledger_hash, block['ledger_hash'],
previous_txlist_hash, block['txlist_hash'])
except ConsensusError as e:
message = str(e)
if message.find('ledger_hash') != -1:
new_ledger = get_block_ledger(memory_db, block['block_index'])
old_ledger = get_block_ledger(prod_db, block['block_index'])
compare_strings(old_ledger, new_ledger)
elif message.find('txlist_hash') != -1:
new_txlist = get_block_txlist(memory_db, block['block_index'])
old_txlist = get_block_txlist(prod_db, block['block_index'])
compare_strings(old_txlist, new_txlist)
raise(e)
def setup_function(function):
counterpartyd.set_options(rpc_port=9999, data_dir=tempfile.gettempdir(), database_file=tempfile.gettempdir()+'/counterpartyd.unittest.db',
rpc_password='******', backend_rpc_password='******',
testnet=True, testcoin=False, backend_rpc_ssl_verify=False)
try:
os.remove(config.DATABASE)
except:
pass
# Connect to database.
global db
db = database.get_connection(read_only=False, foreign_keys=False)
from lib import blocks
blocks.initialise(db)
def setup_module():
counterpartyd.set_options(database_file=tempfile.gettempdir() + '/fixtures.unittest.db', testnet=True, **util_test.COUNTERPARTYD_OPTIONS)
util_test.restore_database(config.DATABASE, CURR_DIR + '/fixtures/scenarios/unittest_fixture.sql')
util.FIRST_MULTISIG_BLOCK_TESTNET = 1
# start RPC server
api_server = api.APIServer()
api_server.daemon = True
api_server.start()
for attempt in range(5000): # wait until server is ready.
if api_server.is_ready:
break
elif attempt == 4999:
raise Exception("Timeout: RPC server not ready after 5s")
else:
time.sleep(0.001)
def do_book(testnet):
# Filenames.
if testnet:
filename = 'book.testnet'
else:
filename = 'book.mainnet'
old = CURR_DIR + '/' + filename
new = old + '.new'
# Get last block_index of old book.
with open(old, 'r') as f:
block_index = int(f.readlines()[-1][7:13])
# Use temporary DB.
counterpartyd.set_options(testnet=testnet)
default_db = config.DATABASE
temp_db = tempfile.gettempdir() + '/' + os.path.basename(config.DATABASE)
shutil.copyfile(default_db, temp_db)
counterpartyd.set_options(database_file=temp_db, testnet=testnet)
db = util.connect_to_db()
cursor = db.cursor()
# TODO: USE API
import subprocess
if testnet:
subprocess.check_call(['./counterpartyd.py', '--database-file=' + temp_db, '--testnet', 'reparse'])
else:
subprocess.check_call(['./counterpartyd.py', '--database-file=' + temp_db, 'reparse'])
# Get new book.
with open(new, 'w') as f:
# Credits.
cursor.execute('select * from credits where block_index <= ? order by block_index, address, asset', (block_index,))
for credit in list(cursor):
f.write('credit ' + str(summarise(credit)) + '\n')
# Debits.
cursor.execute('select * from debits where block_index <= ? order by block_index, address, asset', (block_index,))
for debit in cursor.fetchall():
f.write('debit ' + str(summarise(debit)) + '\n')
# Compare books.
compare(filename)
# Clean up.
cursor.close()
os.remove(temp_db)
def run_scenario(scenario, rawtransactions_db):
counterpartyd.set_options(database_file=':memory:',
testnet=True,
**COUNTERPARTYD_OPTIONS)
config.PREFIX = b'TESTXXXX'
util.FIRST_MULTISIG_BLOCK_TESTNET = 1
checkpoints = dict(check.CHECKPOINTS_TESTNET)
check.CHECKPOINTS_TESTNET = {}
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger_buff = io.StringIO()
handler = logging.StreamHandler(logger_buff)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
asyncio_log = logging.getLogger('asyncio')
asyncio_log.setLevel(logging.ERROR)
db = database.get_connection(read_only=False)
initialise_db(db)
raw_transactions = []
for tx in scenario:
if tx[0] != 'create_next_block':
module = sys.modules['lib.messages.{}'.format(tx[0])]
compose = getattr(module, 'compose')
unsigned_tx_hex = transaction.construct(db, get_proxy(),
compose(db, *tx[1]),
**tx[2])
raw_transactions.append({tx[0]: unsigned_tx_hex})
insert_raw_transaction(unsigned_tx_hex, db, rawtransactions_db)
else:
create_next_block(db,
block_index=config.BURN_START + tx[1],
parse_block=True)
dump = dump_database(db)
log = logger_buff.getvalue()
db.close()
check.CHECKPOINTS_TESTNET = checkpoints
return dump, log, json.dumps(raw_transactions, indent=4)
def setup_function(function):
counterpartyd.set_options(rpc_port=9999,
data_dir=tempfile.gettempdir(),
database_file=tempfile.gettempdir() +
'/counterpartyd.unittest.db',
rpc_password='******',
backend_rpc_password='******',
testnet=True,
testcoin=False,
backend_rpc_ssl_verify=False)
try:
os.remove(config.DATABASE)
except:
pass
# Connect to database.
global db
db = database.get_connection(read_only=False, foreign_keys=False)
from lib import blocks
blocks.initialise(db)
def run_scenario(scenario, rawtransactions_db):
counterpartyd.set_options(database_file=':memory:', testnet=True, **COUNTERPARTYD_OPTIONS)
config.PREFIX = b'TESTXXXX'
util.FIRST_MULTISIG_BLOCK_TESTNET = 1
checkpoints = dict(check.CHECKPOINTS_TESTNET)
check.CHECKPOINTS_TESTNET = {}
proxy = RpcProxy()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger_buff = io.StringIO()
handler = logging.StreamHandler(logger_buff)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
asyncio_log = logging.getLogger('asyncio')
asyncio_log.setLevel(logging.ERROR)
db = database.get_connection(read_only=False)
initialise_db(db)
raw_transactions = []
for transaction in scenario:
if transaction[0] != 'create_next_block':
module = sys.modules['lib.messages.{}'.format(transaction[0])]
compose = getattr(module, 'compose')
unsigned_tx_hex = bitcoin.transaction(db, compose(db, *transaction[1]), **transaction[2])
raw_transactions.append({transaction[0]: unsigned_tx_hex})
insert_raw_transaction(unsigned_tx_hex, db, rawtransactions_db)
else:
create_next_block(db, block_index=config.BURN_START + transaction[1], parse_block=True)
dump = dump_database(db)
log = logger_buff.getvalue()
db.close()
check.CHECKPOINTS_TESTNET = checkpoints
return dump, log, json.dumps(raw_transactions, indent=4)
CURR_DIR = os.path.dirname(
os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))))
sys.path.append(os.path.normpath(os.path.join(CURR_DIR, '..')))
from lib import (config, api, util, exceptions, bitcoin, blocks)
from lib import (send, order, btcpay, issuance, broadcast, bet, dividend, burn,
cancel, callback, rps, rpsresolve)
import counterpartyd
# config.BLOCK_FIRST = 0
# config.BURN_START = 0
# config.BURN_END = 9999999
counterpartyd.set_options(rpc_port=9999,
database_file=CURR_DIR +
'/counterpartyd.unittest.db',
testnet=True,
testcoin=False,
unittest=True)
# unit tests private keys
config.UNITTEST_PRIVKEY = {
'mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc':
'cPdUqd5EbBWsjcG9xiL1hz8bEyGFiz4SW99maU9JgpL9TEcxUf3j',
'mtQheFaSfWELRB2MyMBaiWjdDm6ux9Ezns':
'cQ897jnCVRrawNbw8hgmjMiRNHejwzg4KbzdMCzc91iaTif8ReqX'
}
# Connect to database.
try:
os.remove(config.DATABASE)
except:
# Set test environment
os.environ['TZ'] = 'EST'
time.tzset()
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
CURR_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))))
sys.path.append(os.path.normpath(os.path.join(CURR_DIR, '..')))
from lib import (config, api, util, exceptions, bitcoin, blocks)
from lib import (send, order, btcpay, issuance, broadcast, bet, dividend, burn, cancel, callback, rps, rpsresolve)
import counterpartyd
# config.BLOCK_FIRST = 0
# config.BURN_START = 0
# config.BURN_END = 9999999
counterpartyd.set_options(rpc_port=9999, database_file=CURR_DIR+'/counterpartyd.unittest.db', testnet=True, testcoin=False, unittest=True, backend_rpc_ssl_verify=False)
# unit tests private keys
config.UNITTEST_PRIVKEY = {
'mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc': 'cPdUqd5EbBWsjcG9xiL1hz8bEyGFiz4SW99maU9JgpL9TEcxUf3j',
'mtQheFaSfWELRB2MyMBaiWjdDm6ux9Ezns': 'cQ897jnCVRrawNbw8hgmjMiRNHejwzg4KbzdMCzc91iaTif8ReqX'
}
# Logs.
try: os.remove(CURR_DIR + '/log.new')
except: pass
logging.basicConfig(filename=CURR_DIR + '/log.new', level=logging.DEBUG, format='%(message)s')
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
# Output.
# Set test environment os.environ['TZ'] = 'EST' time.tzset() locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') CURR_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__)))) sys.path.append(os.path.normpath(os.path.join(CURR_DIR, '..'))) from lib import (config, api, util, exceptions, bitcoin, blocks) from lib import (send, order, btcpay, issuance, broadcast, bet, dividend, burn, cancel, callback) import counterpartyd # config.BLOCK_FIRST = 0 # config.BURN_START = 0 # config.BURN_END = 9999999 counterpartyd.set_options(rpc_port=9999, database_file=CURR_DIR+'/counterpartyd.unittest.db', testnet=True, testcoin=False, unittest=True, force=True) # Connect to database. try: os.remove(config.DATABASE) except: pass db = util.connect_to_db() cursor = db.cursor() # Each tx has a block_index equal to its tx_index tx_index = 0 source_default = 'mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc' destination_default = 'n3BrDB6zDiEPWEE6wLxywFb4Yp9ZY5fHM7' quantity = config.UNIT small = round(quantity / 2) expiration = 10
def reparse(testnet=True):
options = dict(COUNTERPARTYD_OPTIONS)
options.pop('data_dir')
counterpartyd.set_options(database_file=':memory:', testnet=testnet, **options)
if testnet:
config.PREFIX = b'TESTXXXX'
logger = logging.getLogger()
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logger.addHandler(console)
memory_db = database.get_connection(read_only=False)
initialise_db(memory_db)
prod_db_path = os.path.join(config.DATA_DIR, '{}.{}{}.db'.format(config.XCP_CLIENT, str(config.VERSION_MAJOR), '.testnet' if testnet else ''))
prod_db = apsw.Connection(prod_db_path)
prod_db.setrowtrace(database.rowtracer)
with memory_db.backup("main", prod_db, "main") as backup:
backup.step()
# here we don't use block.reparse() because it reparse db in transaction (`with db`)
memory_cursor = memory_db.cursor()
for table in blocks.TABLES + ['balances']:
memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))
# clean consensus hashes if first block hash don't match with checkpoint.
checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
columns = [column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')]
for field in ['ledger_hash', 'txlist_hash']:
if field in columns:
sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(field)
first_hash = list(memory_cursor.execute(sql, (config.BLOCK_FIRST,)))[0][field]
if first_hash != checkpoints[config.BLOCK_FIRST][field]:
logging.info('First hash changed. Cleaning {}.'.format(field))
memory_cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field))
blocks.initialise(memory_db)
previous_ledger_hash = None
previous_txlist_hash = None
memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
for block in memory_cursor.fetchall():
try:
logger.info('Block (re‐parse): {}'.format(str(block['block_index'])))
previous_ledger_hash, previous_txlist_hash = blocks.parse_block(memory_db, block['block_index'], block['block_time'],
previous_ledger_hash, block['ledger_hash'],
previous_txlist_hash, block['txlist_hash'])
except check.ConsensusError as e:
message = str(e)
if message.find('ledger_hash') != -1:
new_ledger = get_block_ledger(memory_db, block['block_index'])
old_ledger = get_block_ledger(prod_db, block['block_index'])
compare_strings(old_ledger, new_ledger)
elif message.find('txlist_hash') != -1:
new_txlist = get_block_txlist(memory_db, block['block_index'])
old_txlist = get_block_txlist(prod_db, block['block_index'])
compare_strings(old_txlist, new_txlist)
raise(e)
import inspect import requests from requests.auth import HTTPBasicAuth import logging CURR_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__)))) sys.path.append(os.path.normpath(os.path.join(CURR_DIR, '..'))) from lib import (config, api, util, exceptions, bitcoin, blocks) from lib import (send, order, btcpay, issuance, broadcast, bet, dividend, burn, cancel, callback) import counterpartyd # config.BLOCK_FIRST = 0 # config.BURN_START = 0 # config.BURN_END = 9999999 counterpartyd.set_options(database_file=CURR_DIR+'/counterpartyd.unittest.db', testnet=True, testcoin=False, unittest=True) # Connect to database. try: os.remove(config.DATABASE) except: pass db = util.connect_to_db() cursor = db.cursor() # Each tx has a block_index equal to its tx_index tx_index = 0 source_default = 'mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc' destination_default = 'n3BrDB6zDiEPWEE6wLxywFb4Yp9ZY5fHM7' quantity = config.UNIT small = round(quantity / 2) expiration = 10
import decimal
import time
import json
import logging
from bottle import route, run, template, Bottle, request, static_file, redirect, error, hook, response, abort
import boottleconf
sys.path.insert(0, boottleconf.COUNTERPARTYD_DIR)
from lib import (config, api, util, exceptions, bitcoin, blocks)
from lib import (send, order, btcpay, issuance, broadcast, bet, dividend, burn, cancel, callback)
from counterpartyd import set_options
D = decimal.Decimal
set_options()
db = util.connect_to_db()
app = Bottle()
def S(value):
return int(D(value)*config.UNIT)
class DecimalEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, decimal.Decimal):
return str(o)
return super(DecimalEncoder, self).default(o)
@app.route('/static/<filename:path>')
def reparse(testnet=True):
options = dict(COUNTERPARTYD_OPTIONS)
options.pop('data_dir')
counterpartyd.set_options(database_file=':memory:',
testnet=testnet,
**options)
if testnet:
config.PREFIX = b'TESTXXXX'
logger = logging.getLogger()
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logger.addHandler(console)
memory_db = database.get_connection(read_only=False)
initialise_db(memory_db)
prod_db_path = os.path.join(
config.DATA_DIR, '{}.{}{}.db'.format(config.XCP_CLIENT,
str(config.VERSION_MAJOR),
'.testnet' if testnet else ''))
prod_db = apsw.Connection(prod_db_path)
prod_db.setrowtrace(database.rowtracer)
with memory_db.backup("main", prod_db, "main") as backup:
backup.step()
# here we don't use block.reparse() because it reparse db in transaction (`with db`)
memory_cursor = memory_db.cursor()
for table in blocks.TABLES + ['balances']:
memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))
# clean consensus hashes if first block hash don't match with checkpoint.
checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
columns = [
column['name']
for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')
]
for field in ['ledger_hash', 'txlist_hash']:
if field in columns:
sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(
field)
first_hash = list(
memory_cursor.execute(sql, (config.BLOCK_FIRST, )))[0][field]
if first_hash != checkpoints[config.BLOCK_FIRST][field]:
logger.info('First hash changed. Cleaning {}.'.format(field))
memory_cursor.execute(
'''UPDATE blocks SET {} = NULL'''.format(field))
blocks.initialise(memory_db)
previous_ledger_hash = None
previous_txlist_hash = None
memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
for block in memory_cursor.fetchall():
try:
logger.info('Block (re‐parse): {}'.format(str(
block['block_index'])))
previous_ledger_hash, previous_txlist_hash = blocks.parse_block(
memory_db, block['block_index'], block['block_time'],
previous_ledger_hash, block['ledger_hash'],
previous_txlist_hash, block['txlist_hash'])
except check.ConsensusError as e:
message = str(e)
if message.find('ledger_hash') != -1:
new_ledger = get_block_ledger(memory_db, block['block_index'])
old_ledger = get_block_ledger(prod_db, block['block_index'])
compare_strings(old_ledger, new_ledger)
elif message.find('txlist_hash') != -1:
new_txlist = get_block_txlist(memory_db, block['block_index'])
old_txlist = get_block_txlist(prod_db, block['block_index'])
compare_strings(old_txlist, new_txlist)
raise (e)
