def test_check_database_version():
    server.initialise(
        database_file=tempfile.gettempdir() + "/fixtures.unittest.db", testnet=True, **util_test.COUNTERPARTYD_OPTIONS
    )
    util_test.restore_database(config.DATABASE, CURR_DIR + "/fixtures/scenarios/unittest_fixture.sql")
    db = database.get_connection(read_only=False)
    database.update_version(db)

    version_major, version_minor = database.version(db)
    assert config.VERSION_MAJOR == version_major
    assert config.VERSION_MINOR == version_minor

    check.database_version(db)

    config.VERSION_MINOR += 1
    with pytest.raises(check.DatabaseVersionError) as exception:
        check.database_version(db)
    assert exception.value.reparse_block_index == None
    config.VERSION_MINOR -= 1

    config.VERSION_MAJOR += 1
    with pytest.raises(check.DatabaseVersionError) as exception:
        check.database_version(db)
    assert exception.value.reparse_block_index == config.BLOCK_FIRST
    config.VERSION_MAJOR -= 1
예제 #2
0
def test_check_database_version():
    server.initialise(database_file=tempfile.gettempdir() +
                      '/fixtures.unittest.db',
                      testnet=True,
                      **util_test.COUNTERPARTYD_OPTIONS)
    util_test.restore_database(
        config.DATABASE, CURR_DIR + '/fixtures/scenarios/unittest_fixture.sql')
    db = database.get_connection(read_only=False)
    database.update_version(db)

    version_major, version_minor = database.version(db)
    assert config.VERSION_MAJOR == version_major
    assert config.VERSION_MINOR == version_minor

    check.database_version(db)

    config.VERSION_MINOR += 1
    with pytest.raises(check.DatabaseVersionError) as exception:
        check.database_version(db)
    assert exception.value.reparse_block_index == None
    config.VERSION_MINOR -= 1

    config.VERSION_MAJOR += 1
    with pytest.raises(check.DatabaseVersionError) as exception:
        check.database_version(db)
    assert exception.value.reparse_block_index == config.BLOCK_FIRST
    config.VERSION_MAJOR -= 1
예제 #3
0
def initialise_rawtransactions_db(db):
    """Drop old raw transaction table, create new one and populate it from unspent_outputs.json."""
    if pytest.config.option.savescenarios:
        server.initialise(database_file=':memory:', testnet=True, **COUNTERPARTYD_OPTIONS)
        cursor = db.cursor()
        cursor.execute('DROP TABLE  IF EXISTS raw_transactions')
        cursor.execute('CREATE TABLE IF NOT EXISTS raw_transactions(tx_hash TEXT UNIQUE, tx_hex TEXT)')
        with open(CURR_DIR + '/fixtures/unspent_outputs.json', 'r') as listunspent_test_file:
                wallet_unspent = json.load(listunspent_test_file)
                for output in wallet_unspent:
                    txid = binascii.hexlify(bitcoinlib.core.lx(output['txid'])).decode()
                    tx = backend.deserialize(output['txhex'])
                    cursor.execute('INSERT INTO raw_transactions VALUES (?, ?)', (txid, output['txhex']))
        cursor.close()
예제 #4
0
def initialise_rawtransactions_db(db):
    """Drop old raw transaction table, create new one and populate it from unspent_outputs.json."""
    if pytest.config.option.savescenarios:
        server.initialise(database_file=':memory:', testnet=True, **COUNTERPARTYD_OPTIONS)
        cursor = db.cursor()
        cursor.execute('DROP TABLE  IF EXISTS raw_transactions')
        cursor.execute('CREATE TABLE IF NOT EXISTS raw_transactions(tx_hash TEXT UNIQUE, tx_hex TEXT)')
        with open(CURR_DIR + '/fixtures/unspent_outputs.json', 'r') as listunspent_test_file:
                wallet_unspent = json.load(listunspent_test_file)
                for output in wallet_unspent:
                    txid = binascii.hexlify(bitcoinlib.core.lx(output['txid'])).decode()
                    tx = backend.deserialize(output['txhex'])
                    cursor.execute('INSERT INTO raw_transactions VALUES (?, ?)', (txid, output['txhex']))
        cursor.close()
예제 #5
0
def run_scenario(scenario, rawtransactions_db):
    """Execute a scenario for integration test, returns a dump of the db, a json with raw transactions and the full log."""
    server.initialise(database_file=':memory:',
                      testnet=True,
                      **COUNTERPARTYD_OPTIONS)
    config.PREFIX = b'TESTXXXX'
    util.FIRST_MULTISIG_BLOCK_TESTNET = 1
    checkpoints = dict(check.CHECKPOINTS_TESTNET)
    check.CHECKPOINTS_TESTNET = {}

    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    logger_buff = io.StringIO()
    handler = logging.StreamHandler(logger_buff)
    handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(message)s')
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    requests_log = logging.getLogger("requests")
    requests_log.setLevel(logging.WARNING)
    asyncio_log = logging.getLogger('asyncio')
    asyncio_log.setLevel(logging.ERROR)

    db = database.get_connection(read_only=False)
    initialise_db(db)

    raw_transactions = []
    for tx in scenario:
        if tx[0] != 'create_next_block':
            mock_protocol_changes = tx[3] if len(tx) == 4 else {}
            with MockProtocolChangesContext(**(mock_protocol_changes or {})):
                module = sys.modules['counterpartylib.lib.messages.{}'.format(
                    tx[0])]
                compose = getattr(module, 'compose')
                unsigned_tx_hex = transaction.construct(
                    db, compose(db, *tx[1]), **tx[2])
                raw_transactions.append({tx[0]: unsigned_tx_hex})
                insert_raw_transaction(unsigned_tx_hex, db, rawtransactions_db)
        else:
            create_next_block(db,
                              block_index=config.BURN_START + tx[1],
                              parse_block=tx[2] if len(tx) == 3 else True)

    dump = dump_database(db)
    log = logger_buff.getvalue()

    db.close()
    check.CHECKPOINTS_TESTNET = checkpoints
    return dump, log, json.dumps(raw_transactions, indent=4)
def setup_function(function):
    server.initialise(database_file=tempfile.gettempdir()+'/counterparty.unittest.db', 
                      rpc_port=9999, rpc_password='******', 
                      backend_password='******',
                      testnet=True, testcoin=False)
    try:
        os.remove(config.DATABASE)
    except:
        pass

    # Connect to database.
    global db
    db = database.get_connection(read_only=False, foreign_keys=False)
    from counterpartylib.lib import blocks
    blocks.initialise(db)
예제 #7
0
def init_database(sqlfile, dbfile, options=None):
    kwargs = COUNTERPARTYD_OPTIONS.copy()
    kwargs.update(options or {})

    server.initialise(
        database_file=dbfile,
        testnet=True,
        verbose=True,
        console_logfilter=os.environ.get('COUNTERPARTY_LOGGING', None),
        **kwargs)

    restore_database(config.DATABASE, sqlfile)
    db = database.get_connection(read_only=False)  # reinit the DB to deal with the restoring
    database.update_version(db)
    util.FIRST_MULTISIG_BLOCK_TESTNET = 1

    return db
예제 #8
0
def init_database(sqlfile, dbfile, options=None):
    kwargs = COUNTERPARTYD_OPTIONS.copy()
    kwargs.update(options or {})

    server.initialise(
        database_file=dbfile,
        testnet=True,
        verbose=True,
        console_logfilter=os.environ.get('COUNTERPARTY_LOGGING', None),
        **kwargs)

    restore_database(config.DATABASE, sqlfile)
    db = database.get_connection(read_only=False)  # reinit the DB to deal with the restoring
    database.update_version(db)
    util.FIRST_MULTISIG_BLOCK_TESTNET = 1

    return db
예제 #9
0
def setup_function(function):
    server.initialise(database_file=tempfile.gettempdir() +
                      '/counterparty.unittest.db',
                      rpc_port=9999,
                      rpc_password='******',
                      backend_password='******',
                      testnet=True,
                      testcoin=False)
    try:
        os.remove(config.DATABASE)
    except:
        pass

    # Connect to database.
    global db
    db = database.get_connection(read_only=False, foreign_keys=False)
    from counterpartylib.lib import blocks
    blocks.initialise(db)
예제 #10
0
def run_scenario(scenario):
    """Execute a scenario for integration test, returns a dump of the db, a json with raw transactions and the full log."""
    server.initialise(database_file=':memory:', testnet=True, **COUNTERPARTYD_OPTIONS)
    config.PREFIX = b'TESTXXXX'
    util.FIRST_MULTISIG_BLOCK_TESTNET = 1
    checkpoints = dict(check.CHECKPOINTS_TESTNET)
    check.CHECKPOINTS_TESTNET = {}

    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    logger_buff = io.StringIO()
    handler = logging.StreamHandler(logger_buff)
    handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(message)s')
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    requests_log = logging.getLogger("requests")
    requests_log.setLevel(logging.WARNING)
    asyncio_log = logging.getLogger('asyncio')
    asyncio_log.setLevel(logging.ERROR)

    db = database.get_connection(read_only=False)
    initialise_db(db)

    raw_transactions = []
    for tx in scenario:
        if tx[0] != 'create_next_block':
            mock_protocol_changes = tx[3] if len(tx) == 4 else {}
            with MockProtocolChangesContext(**(mock_protocol_changes or {})):
                module = sys.modules['counterpartylib.lib.messages.{}'.format(tx[0])]
                compose = getattr(module, 'compose')
                unsigned_tx_hex = transaction.construct(db, compose(db, *tx[1]), **tx[2])
                raw_transactions.append({tx[0]: unsigned_tx_hex})
                insert_raw_transaction(unsigned_tx_hex, db)
        else:
            create_next_block(db, block_index=config.BURN_START + tx[1], parse_block=tx[2] if len(tx) == 3 else True)

    dump = dump_database(db)
    log = logger_buff.getvalue()

    db.close()
    check.CHECKPOINTS_TESTNET = checkpoints
    return dump, log, json.dumps(raw_transactions, indent=4)
예제 #11
0
def reparse(testnet=True):
    """
    Reparse all transaction from the database.
     - Create a new in-memory DB, copy the DB that is on-disk
     - Reparse DB, automatically compares consensus hashes to the original ones from the on-disk DB
    """
    options = dict(COUNTERPARTYD_OPTIONS)
    server.initialise(database_file=':memory:', testnet=testnet, **options)

    logger = logging.getLogger()

    if testnet:
        config.PREFIX = b'TESTXXXX'

    memory_db = database.get_connection(read_only=False)

    data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True)
    prod_db_path = os.path.join(data_dir, '{}{}.db'.format(config.APP_NAME, '.testnet' if testnet else ''))
    assert os.path.exists(prod_db_path), "database path {} does not exist".format(prod_db_path)
    prod_db = apsw.Connection(prod_db_path)
    prod_db.setrowtrace(database.rowtracer)

    # Copy DB from file on disk (should be a DB file with at least all the checkpoints)
    #  in-memory DB shouldn't have been written to yet up until this point
    with memory_db.backup("main", prod_db, "main") as backup:
        while not backup.done:
            backup.step(100)

    # Drop most tables (except blocks, transactions, undolog)
    memory_cursor = memory_db.cursor()
    for table in blocks.TABLES + ['balances']:
        memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))

    # Check that all checkpoint blocks are in the database to be tested.
    if testnet:
        CHECKPOINTS = check.CHECKPOINTS_TESTNET
    else:
        CHECKPOINTS = check.CHECKPOINTS_MAINNET
    for block_index in CHECKPOINTS.keys():
        block_exists = bool(list(memory_cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,))))
        assert block_exists, "block #%d does not exist" % block_index

    # Clean consensus hashes if first block hash don’t match with checkpoint.
    checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
    columns = [column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')]
    for field in ['ledger_hash', 'txlist_hash']:
        if field in columns:
            sql = '''SELECT {} FROM blocks  WHERE block_index = ?'''.format(field)
            first_hash = list(memory_cursor.execute(sql, (config.BLOCK_FIRST,)))[0][field]
            if first_hash != checkpoints[config.BLOCK_FIRST][field]:
                logger.info('First hash changed. Cleaning {}.'.format(field))
                memory_cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field))

    # Initialise missing tables
    blocks.initialise(memory_db)
    previous_ledger_hash = None
    previous_txlist_hash = None
    previous_messages_hash = None

    # Reparse each block, if ConsensusError is thrown then the difference
    memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
    for block in memory_cursor.fetchall():
        try:
            util.CURRENT_BLOCK_INDEX = block['block_index']
            previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block(
                                                                     memory_db, block['block_index'], block['block_time'],
                                                                     previous_ledger_hash=previous_ledger_hash, ledger_hash=block['ledger_hash'],
                                                                     previous_txlist_hash=previous_txlist_hash, txlist_hash=block['txlist_hash'],
                                                                     previous_messages_hash=previous_messages_hash)
            logger.info('Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' % (
                block['block_index'], previous_ledger_hash[-5:], previous_txlist_hash[-5:], previous_messages_hash[-5:],
                (' [overwrote %s]' % previous_found_messages_hash) if previous_found_messages_hash and previous_found_messages_hash != previous_messages_hash else ''))

        except check.ConsensusError as e:
            message = str(e)
            if message.find('ledger_hash') != -1:
                new_ledger = get_block_ledger(memory_db, block['block_index'])
                old_ledger = get_block_ledger(prod_db, block['block_index'])
                compare_strings(old_ledger, new_ledger)
            elif message.find('txlist_hash') != -1:
                new_txlist = get_block_txlist(memory_db, block['block_index'])
                old_txlist = get_block_txlist(prod_db, block['block_index'])
                compare_strings(old_txlist, new_txlist)

            raise e
예제 #12
0
def main():
    if os.name == 'nt':
        from counterpartylib.lib import util_windows
        #patch up cmd.exe's "challenged" (i.e. broken/non-existent) UTF-8 logging
        util_windows.fix_win32_unicode()

    # Post installation tasks
    generate_config_files()

    # Parse command-line arguments.
    parser = argparse.ArgumentParser(prog=APP_NAME, description='Server for the {} protocol'.format(config.XCP_NAME), add_help=False)
    parser.add_argument('-h', '--help', dest='help', action='store_true', help='show this help message and exit')
    parser.add_argument('-V', '--version', action='version', version="{} v{}; {} v{}".format(APP_NAME, APP_VERSION, 'counterparty-lib', config.VERSION_STRING))
    parser.add_argument('--config-file', help='the path to the configuration file')

    parser = add_config_arguments(parser, CONFIG_ARGS, 'server.conf')

    subparsers = parser.add_subparsers(dest='action', help='the action to be taken')

    parser_server = subparsers.add_parser('start', help='run the server')

    parser_reparse = subparsers.add_parser('reparse', help='reparse all transactions in the database')
   
    parser_rollback = subparsers.add_parser('rollback', help='rollback database')
    parser_rollback.add_argument('block_index', type=int, help='the index of the last known good block')
    
    parser_kickstart = subparsers.add_parser('kickstart', help='rapidly build database by reading from Bitcoin Core blockchain')
    parser_kickstart.add_argument('--bitcoind-dir', help='Bitcoin Core data directory')

    parser_bootstrap = subparsers.add_parser('bootstrap', help='bootstrap database with hosted snapshot')

    args = parser.parse_args()

    log.set_up(logger, verbose=args.verbose)
    
    logger.info('Running v{} of {}.'.format(APP_VERSION, APP_NAME))

    # Help message
    if args.help:
        parser.print_help()
        sys.exit()

    # Bootstrapping
    if args.action == 'bootstrap':
        bootstrap(testnet=args.testnet)
        sys.exit()

    # Configuration
    if args.action in ['reparse', 'rollback', 'kickstart', 'start']:
        try:
            db = server.initialise(database_file=args.database_file,
                                log_file=args.log_file, api_log_file=args.api_log_file,
                                testnet=args.testnet, testcoin=args.testcoin,
                                backend_name=args.backend_name,
                                backend_connect=args.backend_connect,
                                backend_port=args.backend_port,
                                backend_user=args.backend_user,
                                backend_password=args.backend_password,
                                backend_ssl=args.backend_ssl,
                                backend_ssl_no_verify=args.backend_ssl_no_verify,
                                backend_poll_interval=args.backend_poll_interval,
                                rpc_host=args.rpc_host, rpc_port=args.rpc_port, rpc_user=args.rpc_user,
                                rpc_password=args.rpc_password, rpc_no_allow_cors=args.rpc_no_allow_cors,
                                requests_timeout=args.requests_timeout,
                                rpc_batch_size=args.rpc_batch_size,
                                check_asset_conservation=not args.no_check_asset_conservation,
                                force=args.force, verbose=args.verbose)
                                #,broadcast_tx_mainnet=args.broadcast_tx_mainnet)
        except TypeError as e:
            if 'unexpected keyword argument' in str(e):
                raise VersionError('Unsupported Server Parameter. CLI/Library Version Incompatibility.')
            else:
                raise e

    # PARSING
    if args.action == 'reparse':
        server.reparse(db)

    elif args.action == 'rollback':
        server.reparse(db, block_index=args.block_index)

    elif args.action == 'kickstart':
        server.kickstart(db, bitcoind_dir=args.bitcoind_dir)

    elif args.action == 'start':
        server.start_all(db)

    else:
        parser.print_help()
예제 #13
0
def reparse(testnet=True):
    """Reparse all transaction from the database, create a new blockchain and compare it to the old one."""
    options = dict(COUNTERPARTYD_OPTIONS)
    server.initialise(database_file=':memory:', testnet=testnet, **options)

    logger = logging.getLogger()

    if testnet:
        config.PREFIX = b'TESTXXXX'

    memory_db = database.get_connection(read_only=False)
    initialise_db(memory_db)

    data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME,
                                     appname=config.APP_NAME,
                                     roaming=True)
    prod_db_path = os.path.join(
        data_dir, '{}{}.db'.format(config.APP_NAME,
                                   '.testnet' if testnet else ''))
    assert os.path.exists(
        prod_db_path), "database path {} does not exist".format(prod_db_path)
    prod_db = apsw.Connection(prod_db_path)
    prod_db.setrowtrace(database.rowtracer)

    with memory_db.backup("main", prod_db, "main") as backup:
        backup.step()

    # Here we don’t use block.reparse() because it reparse db in transaction (`with db`).
    memory_cursor = memory_db.cursor()
    for table in blocks.TABLES + ['balances']:
        memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))

    # Check that all checkpoint blocks are in the database to be tested.
    if testnet:
        CHECKPOINTS = check.CHECKPOINTS_TESTNET
    else:
        CHECKPOINTS = check.CHECKPOINTS_MAINNET
    for block_index in CHECKPOINTS.keys():
        block_exists = bool(
            list(
                memory_cursor.execute(
                    '''SELECT * FROM blocks WHERE block_index = ?''',
                    (block_index, ))))
        assert block_exists, "block #%d does not exist" % block_index

    # Clean consensus hashes if first block hash don’t match with checkpoint.
    checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
    columns = [
        column['name']
        for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')
    ]
    for field in ['ledger_hash', 'txlist_hash']:
        if field in columns:
            sql = '''SELECT {} FROM blocks  WHERE block_index = ?'''.format(
                field)
            first_hash = list(
                memory_cursor.execute(sql, (config.BLOCK_FIRST, )))[0][field]
            if first_hash != checkpoints[config.BLOCK_FIRST][field]:
                logger.info('First hash changed. Cleaning {}.'.format(field))
                memory_cursor.execute(
                    '''UPDATE blocks SET {} = NULL'''.format(field))

    blocks.initialise(memory_db)
    previous_ledger_hash = None
    previous_txlist_hash = None
    previous_messages_hash = None

    memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
    for block in memory_cursor.fetchall():
        try:
            util.CURRENT_BLOCK_INDEX = block['block_index']
            previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block(
                memory_db,
                block['block_index'],
                block['block_time'],
                previous_ledger_hash=previous_ledger_hash,
                ledger_hash=block['ledger_hash'],
                previous_txlist_hash=previous_txlist_hash,
                txlist_hash=block['txlist_hash'],
                previous_messages_hash=previous_messages_hash)
            logger.info(
                'Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' %
                (block['block_index'], previous_ledger_hash[-5:],
                 previous_txlist_hash[-5:], previous_messages_hash[-5:],
                 (' [overwrote %s]' %
                  previous_found_messages_hash) if previous_found_messages_hash
                 and previous_found_messages_hash != previous_messages_hash
                 else ''))

        except check.ConsensusError as e:
            message = str(e)
            if message.find('ledger_hash') != -1:
                new_ledger = get_block_ledger(memory_db, block['block_index'])
                old_ledger = get_block_ledger(prod_db, block['block_index'])
                compare_strings(old_ledger, new_ledger)
            elif message.find('txlist_hash') != -1:
                new_txlist = get_block_txlist(memory_db, block['block_index'])
                old_txlist = get_block_txlist(prod_db, block['block_index'])
                compare_strings(old_txlist, new_txlist)
            raise (e)
예제 #14
0
def reparse(testnet=True):
    """
    Reparse all transaction from the database.
     - Create a new in-memory DB, copy the DB that is on-disk
     - Reparse DB, automatically compares consensus hashes to the original ones from the on-disk DB
    """
    options = dict(COUNTERPARTYD_OPTIONS)
    server.initialise(database_file=':memory:', testnet=testnet, **options)

    logger = logging.getLogger()

    if testnet:
        config.PREFIX = b'TESTXXXX'

    memory_db = database.get_connection(read_only=False)

    data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME,
                                     appname=config.APP_NAME,
                                     roaming=True)
    prod_db_path = os.path.join(
        data_dir, '{}{}.db'.format(config.APP_NAME,
                                   '.testnet' if testnet else ''))
    assert os.path.exists(
        prod_db_path), "database path {} does not exist".format(prod_db_path)
    prod_db = apsw.Connection(prod_db_path)
    prod_db.setrowtrace(database.rowtracer)

    # Copy DB from file on disk (should be a DB file with at least all the checkpoints)
    #  in-memory DB shouldn't have been written to yet up until this point
    with memory_db.backup("main", prod_db, "main") as backup:
        while not backup.done:
            backup.step(100)

    # Drop most tables (except blocks, transactions, undolog)
    memory_cursor = memory_db.cursor()
    for table in blocks.TABLES + ['balances']:
        memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))

    # Check that all checkpoint blocks are in the database to be tested.
    if testnet:
        CHECKPOINTS = check.CHECKPOINTS_TESTNET
    else:
        CHECKPOINTS = check.CHECKPOINTS_MAINNET
    for block_index in CHECKPOINTS.keys():
        block_exists = bool(
            list(
                memory_cursor.execute(
                    '''SELECT * FROM blocks WHERE block_index = ?''',
                    (block_index, ))))
        assert block_exists, "block #%d does not exist" % block_index

    # Clean consensus hashes if first block hash don’t match with checkpoint.
    checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
    columns = [
        column['name']
        for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')
    ]
    for field in ['ledger_hash', 'txlist_hash']:
        if field in columns:
            sql = '''SELECT {} FROM blocks  WHERE block_index = ?'''.format(
                field)
            first_hash = list(
                memory_cursor.execute(sql, (config.BLOCK_FIRST, )))[0][field]
            if first_hash != checkpoints[config.BLOCK_FIRST][field]:
                logger.info('First hash changed. Cleaning {}.'.format(field))
                memory_cursor.execute(
                    '''UPDATE blocks SET {} = NULL'''.format(field))

    # Initialise missing tables
    blocks.initialise(memory_db)
    previous_ledger_hash = None
    previous_txlist_hash = None
    previous_messages_hash = None

    # Reparse each block, if ConsensusError is thrown then the difference
    memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
    for block in memory_cursor.fetchall():
        try:
            util.CURRENT_BLOCK_INDEX = block['block_index']
            previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block(
                memory_db,
                block['block_index'],
                block['block_time'],
                previous_ledger_hash=previous_ledger_hash,
                ledger_hash=block['ledger_hash'],
                previous_txlist_hash=previous_txlist_hash,
                txlist_hash=block['txlist_hash'],
                previous_messages_hash=previous_messages_hash)
            logger.info(
                'Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' %
                (block['block_index'], previous_ledger_hash[-5:],
                 previous_txlist_hash[-5:], previous_messages_hash[-5:],
                 (' [overwrote %s]' %
                  previous_found_messages_hash) if previous_found_messages_hash
                 and previous_found_messages_hash != previous_messages_hash
                 else ''))

        except check.ConsensusError as e:
            message = str(e)
            if message.find('ledger_hash') != -1:
                new_ledger = get_block_ledger(memory_db, block['block_index'])
                old_ledger = get_block_ledger(prod_db, block['block_index'])
                compare_strings(old_ledger, new_ledger)
            elif message.find('txlist_hash') != -1:
                new_txlist = get_block_txlist(memory_db, block['block_index'])
                old_txlist = get_block_txlist(prod_db, block['block_index'])
                compare_strings(old_txlist, new_txlist)

            raise e
예제 #15
0
def reparse(testnet=True):
    """Reparse all transaction from the database, create a new blockchain and compare it to the old one."""
    options = dict(COUNTERPARTYD_OPTIONS)
    server.initialise(database_file=':memory:', testnet=testnet, **options)

    logger = logging.getLogger()

    if testnet:
        config.PREFIX = b'TESTXXXX'

    memory_db = database.get_connection(read_only=False)
    initialise_db(memory_db)

    data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True)
    prod_db_path = os.path.join(data_dir, '{}{}.db'.format(config.APP_NAME, '.testnet' if testnet else ''))
    assert os.path.exists(prod_db_path), "database path {} does not exist".format(prod_db_path)
    prod_db = apsw.Connection(prod_db_path)
    prod_db.setrowtrace(database.rowtracer)

    with memory_db.backup("main", prod_db, "main") as backup:
        backup.step()

    # Here we don’t use block.reparse() because it reparse db in transaction (`with db`).
    memory_cursor = memory_db.cursor()
    for table in blocks.TABLES + ['balances']:
        memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))

    # Check that all checkpoint blocks are in the database to be tested.
    if testnet:
        CHECKPOINTS = check.CHECKPOINTS_TESTNET
    else:
        CHECKPOINTS = check.CHECKPOINTS_MAINNET
    for block_index in CHECKPOINTS.keys():
        block_exists = bool(list(memory_cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,))))
        assert block_exists, "block #%d does not exist" % block_index

    # Clean consensus hashes if first block hash don’t match with checkpoint.
    checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
    columns = [column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')]
    for field in ['ledger_hash', 'txlist_hash']:
        if field in columns:
            sql = '''SELECT {} FROM blocks  WHERE block_index = ?'''.format(field)
            first_hash = list(memory_cursor.execute(sql, (config.BLOCK_FIRST,)))[0][field]
            if first_hash != checkpoints[config.BLOCK_FIRST][field]:
                logger.info('First hash changed. Cleaning {}.'.format(field))
                memory_cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field))

    blocks.initialise(memory_db)
    previous_ledger_hash = None
    previous_txlist_hash = None
    previous_messages_hash = None

    memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
    for block in memory_cursor.fetchall():
        try:
            util.CURRENT_BLOCK_INDEX = block['block_index']
            previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block(
                                                                     memory_db, block['block_index'], block['block_time'],
                                                                     previous_ledger_hash=previous_ledger_hash, ledger_hash=block['ledger_hash'],
                                                                     previous_txlist_hash=previous_txlist_hash, txlist_hash=block['txlist_hash'],
                                                                     previous_messages_hash=previous_messages_hash)
            logger.info('Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' % (
                block['block_index'], previous_ledger_hash[-5:], previous_txlist_hash[-5:], previous_messages_hash[-5:],
                (' [overwrote %s]' % previous_found_messages_hash) if previous_found_messages_hash and previous_found_messages_hash != previous_messages_hash else ''))

        except check.ConsensusError as e:
            message = str(e)
            if message.find('ledger_hash') != -1:
                new_ledger = get_block_ledger(memory_db, block['block_index'])
                old_ledger = get_block_ledger(prod_db, block['block_index'])
                compare_strings(old_ledger, new_ledger)
            elif message.find('txlist_hash') != -1:
                new_txlist = get_block_txlist(memory_db, block['block_index'])
                old_txlist = get_block_txlist(prod_db, block['block_index'])
                compare_strings(old_txlist, new_txlist)
            raise(e)
예제 #16
0
def main():
    if os.name == 'nt':
        from counterpartylib.lib import util_windows
        #patch up cmd.exe's "challenged" (i.e. broken/non-existent) UTF-8 logging
        util_windows.fix_win32_unicode()

    # Post installation tasks
    generate_config_files()

    # Parse command-line arguments.
    parser = argparse.ArgumentParser(
        prog=APP_NAME,
        description='Server for the {} protocol'.format(config.XCP_NAME),
        add_help=False)
    parser.add_argument('-h',
                        '--help',
                        dest='help',
                        action='store_true',
                        help='show this help message and exit')
    parser.add_argument('-V',
                        '--version',
                        action='version',
                        version="{} v{}; {} v{}".format(
                            APP_NAME, APP_VERSION, 'counterparty-lib',
                            config.VERSION_STRING))
    parser.add_argument('--config-file',
                        help='the path to the configuration file')

    parser = add_config_arguments(parser, CONFIG_ARGS, 'server.conf')

    subparsers = parser.add_subparsers(dest='action',
                                       help='the action to be taken')

    parser_server = subparsers.add_parser('start', help='run the server')

    parser_reparse = subparsers.add_parser(
        'reparse', help='reparse all transactions in the database')

    parser_rollback = subparsers.add_parser('rollback',
                                            help='rollback database')
    parser_rollback.add_argument('block_index',
                                 type=int,
                                 help='the index of the last known good block')

    parser_kickstart = subparsers.add_parser(
        'kickstart',
        help='rapidly build database by reading from Bitcoin Core blockchain')
    parser_kickstart.add_argument('--bitcoind-dir',
                                  help='Bitcoin Core data directory')

    parser_bootstrap = subparsers.add_parser(
        'bootstrap', help='bootstrap database with hosted snapshot')

    args = parser.parse_args()

    log.set_up(logger, verbose=args.verbose)

    logger.info('Running v{} of {}.'.format(APP_VERSION, APP_NAME))

    # Help message
    if args.help:
        parser.print_help()
        sys.exit()

    # Bootstrapping
    if args.action == 'bootstrap':
        bootstrap(testnet=args.testnet)
        sys.exit()

    # Configuration
    if args.action in ['reparse', 'rollback', 'kickstart', 'start']:
        try:
            db = server.initialise(
                database_file=args.database_file,
                log_file=args.log_file,
                api_log_file=args.api_log_file,
                testnet=args.testnet,
                testcoin=args.testcoin,
                backend_name=args.backend_name,
                backend_connect=args.backend_connect,
                backend_port=args.backend_port,
                backend_user=args.backend_user,
                backend_password=args.backend_password,
                backend_ssl=args.backend_ssl,
                backend_ssl_no_verify=args.backend_ssl_no_verify,
                backend_poll_interval=args.backend_poll_interval,
                rpc_host=args.rpc_host,
                rpc_port=args.rpc_port,
                rpc_user=args.rpc_user,
                rpc_password=args.rpc_password,
                rpc_no_allow_cors=args.rpc_no_allow_cors,
                force=args.force,
                verbose=args.verbose)
            #,broadcast_tx_mainnet=args.broadcast_tx_mainnet)
        except TypeError as e:
            if 'unexpected keyword argument' in str(e):
                raise VersionError(
                    'Unsupported Server Parameter. CLI/Library Version Incompatibility.'
                )
            else:
                raise e

    # PARSING
    if args.action == 'reparse':
        server.reparse(db)

    elif args.action == 'rollback':
        server.reparse(db, block_index=args.block_index)

    elif args.action == 'kickstart':
        server.kickstart(db, bitcoind_dir=args.bitcoind_dir)

    elif args.action == 'start':
        server.start_all(db)

    else:
        parser.print_help()