def run(self): logger.debug('Starting API Status Poller.') global current_api_status_code, current_api_status_response_json db = database.get_connection(read_only=True, integrity_check=False) while self.stop_event.is_set() != True: try: # Check that backend is running, communicable, and caught up with the blockchain. # Check that the database has caught up with bitcoind. if time.time() - self.last_database_check > 10 * 60: # Ten minutes since last check. if not config.FORCE: code = 11 logger.debug('Checking backend state.') check_backend_state() code = 12 logger.debug('Checking database state.') check_database_state(db, backend.getblockcount()) self.last_database_check = time.time() except (BackendError, DatabaseError) as e: exception_name = e.__class__.__name__ exception_text = str(e) logger.debug("API Status Poller: %s", exception_text) jsonrpc_response = jsonrpc.exceptions.JSONRPCServerError(message=exception_name, data=exception_text) current_api_status_code = code current_api_status_response_json = jsonrpc_response.json.encode() else: current_api_status_code = None current_api_status_response_json = None time.sleep(config.BACKEND_POLL_INTERVAL)
def test_check_database_version(): server.initialise(database_file=tempfile.gettempdir() + '/fixtures.unittest.db', testnet=True, **util_test.COUNTERPARTYD_OPTIONS) util_test.restore_database( config.DATABASE, CURR_DIR + '/fixtures/scenarios/unittest_fixture.sql') db = database.get_connection(read_only=False) database.update_version(db) version_major, version_minor = database.version(db) assert config.VERSION_MAJOR == version_major assert config.VERSION_MINOR == version_minor check.database_version(db) config.VERSION_MINOR += 1 with pytest.raises(check.DatabaseVersionError) as exception: check.database_version(db) assert exception.value.reparse_block_index == None config.VERSION_MINOR -= 1 config.VERSION_MAJOR += 1 with pytest.raises(check.DatabaseVersionError) as exception: check.database_version(db) assert exception.value.reparse_block_index == config.BLOCK_FIRST config.VERSION_MAJOR -= 1
def test_check_database_version(): server.initialise( database_file=tempfile.gettempdir() + "/fixtures.unittest.db", testnet=True, **util_test.COUNTERPARTYD_OPTIONS ) util_test.restore_database(config.DATABASE, CURR_DIR + "/fixtures/scenarios/unittest_fixture.sql") db = database.get_connection(read_only=False) database.update_version(db) version_major, version_minor = database.version(db) assert config.VERSION_MAJOR == version_major assert config.VERSION_MINOR == version_minor check.database_version(db) config.VERSION_MINOR += 1 with pytest.raises(check.DatabaseVersionError) as exception: check.database_version(db) assert exception.value.reparse_block_index == None config.VERSION_MINOR -= 1 config.VERSION_MAJOR += 1 with pytest.raises(check.DatabaseVersionError) as exception: check.database_version(db) assert exception.value.reparse_block_index == config.BLOCK_FIRST config.VERSION_MAJOR -= 1
def run(self): logger.debug('Starting API Status Poller.') global current_api_status_code, current_api_status_response_json db = database.get_connection(read_only=True, integrity_check=False) while self.stop_event.is_set() != True: try: # Check that bitcoind is running, communicable, and caught up with the blockchain. # Check that the database has caught up with bitcoind. if time.time( ) - self.last_database_check > 10 * 60: # Ten minutes since last check. code = 11 logger.debug('Checking backend state.') check.backend_state() code = 12 logger.debug('Checking database state.') check.database_state(db, backend.getblockcount()) self.last_database_check = time.time() except (check.BackendError, check.DatabaseError) as e: exception_name = e.__class__.__name__ exception_text = str(e) logger.debug("API Status Poller: %s", exception_text) jsonrpc_response = jsonrpc.exceptions.JSONRPCServerError( message=exception_name, data=exception_text) current_api_status_code = code current_api_status_response_json = jsonrpc_response.json.encode( ) else: current_api_status_code = None current_api_status_response_json = None time.sleep(config.BACKEND_POLL_INTERVAL)
def counterpartyd_db(request): """Enable database access for unit test vectors.""" db = database.get_connection(read_only=False) database.update_version(db) cursor = db.cursor() cursor.execute('''BEGIN''') request.addfinalizer(lambda: cursor.execute('''ROLLBACK''')) return db
def server_db(request): """Enable database access for unit test vectors.""" db = database.get_connection(read_only=False) database.update_version(db) cursor = db.cursor() cursor.execute('''BEGIN''') request.addfinalizer(lambda: cursor.execute('''ROLLBACK''')) return db
def server_db(request, cp_server): """Enable database access for unit test vectors.""" db = database.get_connection(read_only=False, integrity_check=False) cursor = db.cursor() cursor.execute('''BEGIN''') util_test.reset_current_block_index(db) request.addfinalizer(lambda: cursor.execute('''ROLLBACK''')) request.addfinalizer(lambda: util_test.reset_current_block_index(db)) return db
def server_db(request, cp_server): """Enable database access for unit test vectors.""" db = database.get_connection(read_only=False, integrity_check=False) cursor = db.cursor() cursor.execute('''BEGIN''') util_test.reset_current_block_index(db) request.addfinalizer(lambda: cursor.execute('''ROLLBACK''')) request.addfinalizer(lambda: util_test.reset_current_block_index(db)) return db
def run_scenario(scenario, rawtransactions_db): """Execute a scenario for integration test, returns a dump of the db, a json with raw transactions and the full log.""" server.initialise(database_file=':memory:', testnet=True, **COUNTERPARTYD_OPTIONS) config.PREFIX = b'TESTXXXX' util.FIRST_MULTISIG_BLOCK_TESTNET = 1 checkpoints = dict(check.CHECKPOINTS_TESTNET) check.CHECKPOINTS_TESTNET = {} logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger_buff = io.StringIO() handler = logging.StreamHandler(logger_buff) handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) logger.addHandler(handler) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) asyncio_log = logging.getLogger('asyncio') asyncio_log.setLevel(logging.ERROR) db = database.get_connection(read_only=False) initialise_db(db) raw_transactions = [] for tx in scenario: if tx[0] != 'create_next_block': mock_protocol_changes = tx[3] if len(tx) == 4 else {} with MockProtocolChangesContext(**(mock_protocol_changes or {})): module = sys.modules['counterpartylib.lib.messages.{}'.format( tx[0])] compose = getattr(module, 'compose') unsigned_tx_hex = transaction.construct( db, compose(db, *tx[1]), **tx[2]) raw_transactions.append({tx[0]: unsigned_tx_hex}) insert_raw_transaction(unsigned_tx_hex, db, rawtransactions_db) else: create_next_block(db, block_index=config.BURN_START + tx[1], parse_block=tx[2] if len(tx) == 3 else True) dump = dump_database(db) log = logger_buff.getvalue() db.close() check.CHECKPOINTS_TESTNET = checkpoints return dump, log, json.dumps(raw_transactions, indent=4)
def initialise_db(): if config.FORCE: logger.warning('THE OPTION `--force` IS NOT FOR USE ON PRODUCTION SYSTEMS.') # Lock if not config.FORCE: get_lock() # Database logger.info('Connecting to database (SQLite %s).' % apsw.apswversion()) db = database.get_connection(read_only=False) util.CURRENT_BLOCK_INDEX = blocks.last_db_index(db) return db
def initialise_db(): if config.FORCE: logger.warning('THE OPTION `--force` IS NOT FOR USE ON PRODUCTION SYSTEMS.') # Lock if not config.FORCE: get_lock() # Database logger.info('Connecting to database.') db = database.get_connection(read_only=False) util.CURRENT_BLOCK_INDEX = blocks.last_db_index(db) return db
def setup_function(function): server.initialise(database_file=tempfile.gettempdir()+'/counterparty.unittest.db', rpc_port=9999, rpc_password='******', backend_password='******', testnet=True, testcoin=False) try: os.remove(config.DATABASE) except: pass # Connect to database. global db db = database.get_connection(read_only=False, foreign_keys=False) from counterpartylib.lib import blocks blocks.initialise(db)
def init_database(sqlfile, dbfile, options=None): kwargs = COUNTERPARTYD_OPTIONS.copy() kwargs.update(options or {}) server.initialise( database_file=dbfile, testnet=True, verbose=True, console_logfilter=os.environ.get('COUNTERPARTY_LOGGING', None), **kwargs) restore_database(config.DATABASE, sqlfile) db = database.get_connection(read_only=False) # reinit the DB to deal with the restoring database.update_version(db) util.FIRST_MULTISIG_BLOCK_TESTNET = 1 return db
def init_database(sqlfile, dbfile, options=None): kwargs = COUNTERPARTYD_OPTIONS.copy() kwargs.update(options or {}) server.initialise( database_file=dbfile, testnet=True, verbose=True, console_logfilter=os.environ.get('COUNTERPARTY_LOGGING', None), **kwargs) restore_database(config.DATABASE, sqlfile) db = database.get_connection(read_only=False) # reinit the DB to deal with the restoring database.update_version(db) util.FIRST_MULTISIG_BLOCK_TESTNET = 1 return db
def setup_function(function): server.initialise(database_file=tempfile.gettempdir() + '/counterparty.unittest.db', rpc_port=9999, rpc_password='******', backend_password='******', testnet=True, testcoin=False) try: os.remove(config.DATABASE) except: pass # Connect to database. global db db = database.get_connection(read_only=False, foreign_keys=False) from counterpartylib.lib import blocks blocks.initialise(db)
def run_scenario(scenario): """Execute a scenario for integration test, returns a dump of the db, a json with raw transactions and the full log.""" server.initialise(database_file=':memory:', testnet=True, **COUNTERPARTYD_OPTIONS) config.PREFIX = b'TESTXXXX' util.FIRST_MULTISIG_BLOCK_TESTNET = 1 checkpoints = dict(check.CHECKPOINTS_TESTNET) check.CHECKPOINTS_TESTNET = {} logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger_buff = io.StringIO() handler = logging.StreamHandler(logger_buff) handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) logger.addHandler(handler) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) asyncio_log = logging.getLogger('asyncio') asyncio_log.setLevel(logging.ERROR) db = database.get_connection(read_only=False) initialise_db(db) raw_transactions = [] for tx in scenario: if tx[0] != 'create_next_block': mock_protocol_changes = tx[3] if len(tx) == 4 else {} with MockProtocolChangesContext(**(mock_protocol_changes or {})): module = sys.modules['counterpartylib.lib.messages.{}'.format(tx[0])] compose = getattr(module, 'compose') unsigned_tx_hex = transaction.construct(db, compose(db, *tx[1]), **tx[2]) raw_transactions.append({tx[0]: unsigned_tx_hex}) insert_raw_transaction(unsigned_tx_hex, db) else: create_next_block(db, block_index=config.BURN_START + tx[1], parse_block=tx[2] if len(tx) == 3 else True) dump = dump_database(db) log = logger_buff.getvalue() db.close() check.CHECKPOINTS_TESTNET = checkpoints return dump, log, json.dumps(raw_transactions, indent=4)
def run(self): logger.info('Starting API Server.') db = database.get_connection(read_only=True, integrity_check=False) app = flask.Flask(__name__) auth = HTTPBasicAuth() @auth.get_password def get_pw(username): if username == config.RPC_USER: return config.RPC_PASSWORD return None ###################### #READ API # Generate dynamically get_{table} methods def generate_get_method(table): def get_method(**kwargs): try: return get_rows(db, table=table, **kwargs) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) return get_method for table in API_TABLES: new_method = generate_get_method(table) new_method.__name__ = 'get_{}'.format(table) dispatcher.add_method(new_method) @dispatcher.add_method def sql(query, bindings=None): if bindings == None: bindings = [] return db_query(db, query, tuple(bindings)) ###################### #WRITE/ACTION API # Generate dynamically create_{transaction} methods def generate_create_method(tx): def split_params(**kwargs): transaction_args = {} common_args = {} private_key_wif = None for key in kwargs: if key in COMMONS_ARGS: common_args[key] = kwargs[key] elif key == 'privkey': private_key_wif = kwargs[key] else: transaction_args[key] = kwargs[key] return transaction_args, common_args, private_key_wif def create_method(**kwargs): try: transaction_args, common_args, private_key_wif = split_params( **kwargs) return compose_transaction(db, name=tx, params=transaction_args, **common_args) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) return create_method for tx in API_TRANSACTIONS: create_method = generate_create_method(tx) create_method.__name__ = 'create_{}'.format(tx) dispatcher.add_method(create_method) @dispatcher.add_method def get_messages(block_index): if not isinstance(block_index, int): raise APIError("block_index must be an integer.") cursor = db.cursor() cursor.execute( 'select * from messages where block_index = ? order by message_index asc', (block_index, )) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_messages_by_index(message_indexes): """Get specific messages from the feed, based on the message_index. @param message_index: A single index, or a list of one or more message indexes to retrieve. """ if not isinstance(message_indexes, list): message_indexes = [ message_indexes, ] for idx in message_indexes: #make sure the data is clean if not isinstance(idx, int): raise APIError( "All items in message_indexes are not integers") cursor = db.cursor() cursor.execute( 'SELECT * FROM messages WHERE message_index IN (%s) ORDER BY message_index ASC' % (','.join([str(x) for x in message_indexes]), )) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_xcp_supply(): return util.xcp_supply(db) @dispatcher.add_method def get_asset_info(assets): if not isinstance(assets, list): raise APIError( "assets must be a list of asset names, even if it just contains one entry" ) assetsInfo = [] for asset in assets: # BTC and XCP. if asset in [config.BTC, config.XCP]: if asset == config.BTC: supply = backend.get_btc_supply(normalize=False) else: supply = util.xcp_supply(db) assetsInfo.append({ 'asset': asset, 'owner': None, 'divisible': True, 'locked': False, 'supply': supply, 'description': '', 'issuer': None }) continue # User‐created asset. cursor = db.cursor() issuances = list( cursor.execute( '''SELECT * FROM issuances WHERE (status = ? AND asset = ?) ORDER BY block_index ASC''', ('valid', asset))) cursor.close() if not issuances: continue #asset not found, most likely else: last_issuance = issuances[-1] locked = False for e in issuances: if e['locked']: locked = True assetsInfo.append({ 'asset': asset, 'owner': last_issuance['issuer'], 'divisible': bool(last_issuance['divisible']), 'locked': locked, 'supply': util.asset_supply(db, asset), 'description': last_issuance['description'], 'issuer': last_issuance['issuer'] }) return assetsInfo @dispatcher.add_method def get_block_info(block_index): assert isinstance(block_index, int) cursor = db.cursor() cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index, )) blocks = list(cursor) if len(blocks) == 1: block = blocks[0] elif len(blocks) == 0: raise exceptions.DatabaseError('No blocks found.') else: assert False cursor.close() return block @dispatcher.add_method def get_blocks(block_indexes): """fetches block info and messages for the specified block indexes""" if not isinstance(block_indexes, (list, tuple)): raise APIError("block_indexes must be a list of integers.") if len(block_indexes) >= 250: raise APIError("can only specify up to 250 indexes at a time.") block_indexes_str = ','.join([str(x) for x in block_indexes]) cursor = db.cursor() cursor.execute( 'SELECT * FROM blocks WHERE block_index IN (%s) ORDER BY block_index ASC' % (block_indexes_str, )) blocks = cursor.fetchall() cursor.execute( 'SELECT * FROM messages WHERE block_index IN (%s) ORDER BY block_index ASC, message_index ASC' % (block_indexes_str, )) messages = collections.deque(cursor.fetchall()) for block in blocks: # messages_in_block = [] block['_messages'] = [] while len(messages) and messages[0]['block_index'] == block[ 'block_index']: block['_messages'].append(messages.popleft()) assert not len(messages) #should have been cleared out cursor.close() return blocks @dispatcher.add_method def get_running_info(): latestBlockIndex = backend.getblockcount() try: check.database_state(db, latestBlockIndex) except check.DatabaseError: caught_up = False else: caught_up = True try: cursor = db.cursor() blocks = list( cursor.execute( '''SELECT * FROM blocks WHERE block_index = ?''', (util.CURRENT_BLOCK_INDEX, ))) assert len(blocks) == 1 last_block = blocks[0] cursor.close() except: last_block = None try: last_message = util.last_message(db) except: last_message = None return { 'db_caught_up': caught_up, 'bitcoin_block_count': latestBlockIndex, 'last_block': last_block, 'last_message_index': last_message['message_index'] if last_message else -1, 'running_testnet': config.TESTNET, 'running_testcoin': config.TESTCOIN, 'version_major': config.VERSION_MAJOR, 'version_minor': config.VERSION_MINOR, 'version_revision': config.VERSION_REVISION } @dispatcher.add_method def get_element_counts(): counts = {} cursor = db.cursor() for element in [ 'transactions', 'blocks', 'debits', 'credits', 'balances', 'sends', 'orders', 'order_matches', 'btcpays', 'issuances', 'broadcasts', 'bets', 'bet_matches', 'dividends', 'burns', 'cancels', 'order_expirations', 'bet_expirations', 'order_match_expirations', 'bet_match_expirations', 'messages' ]: cursor.execute("SELECT COUNT(*) AS count FROM %s" % element) count_list = cursor.fetchall() assert len(count_list) == 1 counts[element] = count_list[0]['count'] cursor.close() return counts @dispatcher.add_method def get_asset_names(): cursor = db.cursor() names = [ row['asset'] for row in cursor.execute( "SELECT DISTINCT asset FROM issuances WHERE status = 'valid' ORDER BY asset ASC" ) ] cursor.close() return names @dispatcher.add_method def get_holder_count(asset): holders = util.holders(db, asset) addresses = [] for holder in holders: addresses.append(holder['address']) return {asset: len(set(addresses))} @dispatcher.add_method def get_holders(asset): holders = util.holders(db, asset) return holders @dispatcher.add_method def search_raw_transactions(address): return backend.searchrawtransactions(address) @dispatcher.add_method def get_unspent_txouts(address, return_confirmed=False): result = backend.get_unspent_txouts( address, return_confirmed=return_confirmed) if return_confirmed: return {'all': result[0], 'confirmed': result[1]} else: return result @dispatcher.add_method def get_tx_info(tx_hex): source, destination, btc_amount, fee, data = blocks.get_tx_info( tx_hex) return source, destination, btc_amount, fee, util.hexlify(data) @dispatcher.add_method def unpack(data_hex): data = binascii.unhexlify(data_hex) message_type_id = struct.unpack(config.TXTYPE_FORMAT, data[:4])[0] message = data[4:] # TODO: This works for precisely those messages for which # `unpack()` is defined. for message_type in API_TRANSACTIONS: if message_type_id == sys.modules['lib.messages.{}'.format( message_type)].ID: unpack_method = sys.modules['lib.messages.{}'.format( message_type)].unpack unpacked = unpack_method(db, message, util.CURRENT_BLOCK_INDEX) return message_type_id, unpacked @dispatcher.add_method # TODO: Rename this method. def search_pubkey(pubkeyhash, provided_pubkeys=None): # Returns `None` if the public key cannot be found. return backend.pubkeyhash_to_pubkey( pubkeyhash, provided_pubkeys=provided_pubkeys) def _set_cors_headers(response): if config.RPC_ALLOW_CORS: response.headers['Access-Control-Allow-Origin'] = '*' response.headers[ 'Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS' response.headers[ 'Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type' @app.route('/', methods=[ "OPTIONS", ]) @app.route('/api/', methods=[ "OPTIONS", ]) def handle_options(): response = flask.Response('', 204) _set_cors_headers(response) return response @app.route('/', methods=[ "POST", ]) @app.route('/api/', methods=[ "POST", ]) @auth.login_required def handle_post(): # Check for valid request format. try: request_json = flask.request.get_data().decode('utf-8') request_data = json.loads(request_json) assert 'id' in request_data and request_data[ 'jsonrpc'] == "2.0" and request_data['method'] # params may be omitted except: obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest( data="Invalid JSON-RPC 2.0 request format") return flask.Response(obj_error.json.encode(), 400, mimetype='application/json') # Only arguments passed as a `dict` are supported. if request_data.get('params', None) and not isinstance( request_data['params'], dict): obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest( data= 'Arguments must be passed as a JSON object (list of unnamed arguments not supported)' ) return flask.Response(obj_error.json.encode(), 400, mimetype='application/json') # Return an error if the API Status Poller checks fail. if not config.FORCE and current_api_status_code: return flask.Response(current_api_status_response_json, 503, mimetype='application/json') # Answer request normally. jsonrpc_response = jsonrpc.JSONRPCResponseManager.handle( request_json, dispatcher) response = flask.Response(jsonrpc_response.json.encode(), 200, mimetype='application/json') _set_cors_headers(response) return response init_api_access_log() http_server = HTTPServer(WSGIContainer(app), xheaders=True) try: http_server.listen(config.RPC_PORT, address=config.RPC_HOST) self.is_ready = True self.ioloop.start() except OSError: raise APIError( "Cannot start the API subsystem. Is server already running, or is something else listening on port {}?" .format(config.RPC_PORT)) db.close() http_server.stop() self.ioloop.close() return
def initialise( database_file=None, log_file=None, api_log_file=None, testnet=False, testcoin=False, backend_name=None, backend_connect=None, backend_port=None, backend_user=None, backend_password=None, backend_ssl=False, backend_ssl_no_verify=False, backend_poll_interval=None, rpc_host=None, rpc_port=None, rpc_user=None, rpc_password=None, rpc_no_allow_cors=False, force=False, verbose=False, requests_timeout=config.DEFAULT_REQUESTS_TIMEOUT, rpc_batch_size=config.DEFAULT_RPC_BATCH_SIZE, check_asset_conservation=config.DEFAULT_CHECK_ASSET_CONSERVATION, backend_ssl_verify=None, rpc_allow_cors=None, ): # Data directory data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) if not os.path.isdir(data_dir): os.makedirs(data_dir, mode=0o755) # testnet if testnet: config.TESTNET = testnet else: config.TESTNET = False # testcoin if testcoin: config.TESTCOIN = testcoin else: config.TESTCOIN = False network = "" if config.TESTNET: network += ".testnet" if config.TESTCOIN: network += ".testcoin" # Database if database_file: config.DATABASE = database_file else: filename = "{}{}.db".format(config.APP_NAME, network) config.DATABASE = os.path.join(data_dir, filename) # Log directory log_dir = appdirs.user_log_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME) if not os.path.isdir(log_dir): os.makedirs(log_dir, mode=0o755) # Log if log_file: config.LOG = log_file else: filename = "server{}.log".format(network) config.LOG = os.path.join(log_dir, filename) logger.debug("Writing server log to file: `{}`".format(config.LOG)) if api_log_file: config.API_LOG = api_log_file else: filename = "server{}.access.log".format(network) config.API_LOG = os.path.join(log_dir, filename) logger.debug("Writing API accesses log to file: `{}`".format(config.API_LOG)) # Set up logging. root_logger = logging.getLogger() # Get root logger. log.set_up(root_logger, verbose=verbose, logfile=config.LOG) # Log unhandled errors. def handle_exception(exc_type, exc_value, exc_traceback): logger.error("Unhandled Exception", exc_info=(exc_type, exc_value, exc_traceback)) sys.excepthook = handle_exception ############## # THINGS WE CONNECT TO # Backend name if backend_name: config.BACKEND_NAME = backend_name else: config.BACKEND_NAME = "addrindex" if config.BACKEND_NAME == "jmcorgan": config.BACKEND_NAME = "addrindex" # Backend RPC host (Bitcoin Core) if backend_connect: config.BACKEND_CONNECT = backend_connect else: config.BACKEND_CONNECT = "localhost" # Backend Core RPC port (Bitcoin Core) if backend_port: config.BACKEND_PORT = backend_port else: if config.TESTNET: if config.BACKEND_NAME == "btcd": config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT_TESTNET_BTCD else: config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT_TESTNET else: if config.BACKEND_NAME == "btcd": config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT_BTCD else: config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT try: config.BACKEND_PORT = int(config.BACKEND_PORT) if not (int(config.BACKEND_PORT) > 1 and int(config.BACKEND_PORT) < 65535): raise ConfigurationError("invalid backend API port number") except: raise ConfigurationError("Please specific a valid port number backend-port configuration parameter") # Backend Core RPC user (Bitcoin Core) if backend_user: config.BACKEND_USER = backend_user else: config.BACKEND_USER = "******" # Backend Core RPC password (Bitcoin Core) if backend_password: config.BACKEND_PASSWORD = backend_password else: raise ConfigurationError( "backend RPC password not set. (Use configuration file or --backend-password=PASSWORD)" ) # Backend Core RPC SSL if backend_ssl: config.BACKEND_SSL = backend_ssl else: config.BACKEND_SSL = False # Default to off. # Backend Core RPC SSL Verify if backend_ssl_verify is not None: logger.warning("The server parameter `backend_ssl_verify` is deprecated. Use `backend_ssl_no_verify` instead.") config.BACKEND_SSL_NO_VERIFY = not backend_ssl_verify else: if backend_ssl_no_verify: config.BACKEND_SSL_NO_VERIFY = backend_ssl_no_verify else: config.BACKEND_SSL_NO_VERIFY = False # Default to on (don't support self‐signed certificates) # Backend Poll Interval if backend_poll_interval: config.BACKEND_POLL_INTERVAL = backend_poll_interval else: config.BACKEND_POLL_INTERVAL = 0.5 # Construct backend URL. config.BACKEND_URL = ( config.BACKEND_USER + ":" + config.BACKEND_PASSWORD + "@" + config.BACKEND_CONNECT + ":" + str(config.BACKEND_PORT) ) if config.BACKEND_SSL: config.BACKEND_URL = "https://" + config.BACKEND_URL else: config.BACKEND_URL = "http://" + config.BACKEND_URL ############## # THINGS WE SERVE # Server API RPC host if rpc_host: config.RPC_HOST = rpc_host else: config.RPC_HOST = "localhost" # The web root directory for API calls, eg. localhost:14000/rpc/ config.RPC_WEBROOT = "/rpc/" # Server API RPC port if rpc_port: config.RPC_PORT = rpc_port else: if config.TESTNET: if config.TESTCOIN: config.RPC_PORT = config.DEFAULT_RPC_PORT_TESTNET + 1 else: config.RPC_PORT = config.DEFAULT_RPC_PORT_TESTNET else: if config.TESTCOIN: config.RPC_PORT = config.DEFAULT_RPC_PORT + 1 else: config.RPC_PORT = config.DEFAULT_RPC_PORT try: config.RPC_PORT = int(config.RPC_PORT) if not (int(config.RPC_PORT) > 1 and int(config.RPC_PORT) < 65535): raise ConfigurationError("invalid server API port number") except: raise ConfigurationError("Please specific a valid port number rpc-port configuration parameter") # Server API RPC user if rpc_user: config.RPC_USER = rpc_user else: config.RPC_USER = "******" # Server API RPC password if rpc_password: config.RPC_PASSWORD = rpc_password config.RPC = ( "http://" + urlencode(config.RPC_USER) + ":" + urlencode(config.RPC_PASSWORD) + "@" + config.RPC_HOST + ":" + str(config.RPC_PORT) + config.RPC_WEBROOT ) else: config.RPC = "http://" + config.RPC_HOST + ":" + str(config.RPC_PORT) + config.RPC_WEBROOT # RPC CORS if rpc_allow_cors is not None: logger.warning("The server parameter `rpc_allow_cors` is deprecated. Use `rpc_no_allow_cors` instead.") config.RPC_NO_ALLOW_CORS = not rpc_allow_cors else: if rpc_no_allow_cors: config.RPC_NO_ALLOW_CORS = rpc_no_allow_cors else: config.RPC_NO_ALLOW_CORS = False config.REQUESTS_TIMEOUT = requests_timeout config.RPC_BATCH_SIZE = rpc_batch_size config.CHECK_ASSET_CONSERVATION = check_asset_conservation ############## # OTHER SETTINGS # skip checks if force: config.FORCE = force else: config.FORCE = False # Encoding if config.TESTCOIN: config.PREFIX = b"XX" # 2 bytes (possibly accidentally created) else: config.PREFIX = b"CNTRPRTY" # 8 bytes # (more) Testnet if config.TESTNET: config.MAGIC_BYTES = config.MAGIC_BYTES_TESTNET if config.TESTCOIN: config.ADDRESSVERSION = config.ADDRESSVERSION_TESTNET config.BLOCK_FIRST = config.BLOCK_FIRST_TESTNET_TESTCOIN config.BURN_START = config.BURN_START_TESTNET_TESTCOIN config.BURN_END = config.BURN_END_TESTNET_TESTCOIN config.UNSPENDABLE = config.UNSPENDABLE_TESTNET else: config.ADDRESSVERSION = config.ADDRESSVERSION_TESTNET config.BLOCK_FIRST = config.BLOCK_FIRST_TESTNET config.BURN_START = config.BURN_START_TESTNET config.BURN_END = config.BURN_END_TESTNET config.UNSPENDABLE = config.UNSPENDABLE_TESTNET else: config.MAGIC_BYTES = config.MAGIC_BYTES_MAINNET if config.TESTCOIN: config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET config.BLOCK_FIRST = config.BLOCK_FIRST_MAINNET_TESTCOIN config.BURN_START = config.BURN_START_MAINNET_TESTCOIN config.BURN_END = config.BURN_END_MAINNET_TESTCOIN config.UNSPENDABLE = config.UNSPENDABLE_MAINNET else: config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET config.BLOCK_FIRST = config.BLOCK_FIRST_MAINNET config.BURN_START = config.BURN_START_MAINNET config.BURN_END = config.BURN_END_MAINNET config.UNSPENDABLE = config.UNSPENDABLE_MAINNET logger.info("Running v{} of counterparty-lib.".format(config.VERSION_STRING)) if config.FORCE: logger.warning("THE OPTION `--force` IS NOT FOR USE ON PRODUCTION SYSTEMS.") # Lock if not config.FORCE: get_lock() # Database logger.info("Connecting to database.") db = database.get_connection(read_only=False) util.CURRENT_BLOCK_INDEX = blocks.last_db_index(db) return db
def reparse(testnet=True): """Reparse all transaction from the database, create a new blockchain and compare it to the old one.""" options = dict(COUNTERPARTYD_OPTIONS) server.initialise(database_file=':memory:', testnet=testnet, **options) if testnet: config.PREFIX = b'TESTXXXX' logger = logging.getLogger() console = logging.StreamHandler() console.setLevel(logging.INFO) formatter = logging.Formatter('%(message)s') console.setFormatter(formatter) logger.addHandler(console) memory_db = database.get_connection(read_only=False) initialise_db(memory_db) data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) prod_db_path = os.path.join(data_dir, '{}{}.db'.format(config.APP_NAME, '.testnet' if testnet else '')) prod_db = apsw.Connection(prod_db_path) prod_db.setrowtrace(database.rowtracer) with memory_db.backup("main", prod_db, "main") as backup: backup.step() # Here we don’t use block.reparse() because it reparse db in transaction (`with db`). memory_cursor = memory_db.cursor() for table in blocks.TABLES + ['balances']: memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table)) # Check that all checkpoint blocks are in the database to be tested. if testnet: CHECKPOINTS = check.CHECKPOINTS_TESTNET else: CHECKPOINTS = check.CHECKPOINTS_MAINNET for block_index in CHECKPOINTS.keys(): block_exists = bool(list(memory_cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,)))) assert block_exists # Clean consensus hashes if first block hash don’t match with checkpoint. checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET columns = [column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')] for field in ['ledger_hash', 'txlist_hash']: if field in columns: sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(field) first_hash = list(memory_cursor.execute(sql, (config.BLOCK_FIRST,)))[0][field] if first_hash != checkpoints[config.BLOCK_FIRST][field]: logger.info('First hash changed. Cleaning {}.'.format(field)) memory_cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field)) blocks.initialise(memory_db) previous_ledger_hash = None previous_txlist_hash = None previous_messages_hash = None memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''') for block in memory_cursor.fetchall(): try: util.CURRENT_BLOCK_INDEX = block['block_index'] previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block( memory_db, block['block_index'], block['block_time'], previous_ledger_hash=previous_ledger_hash, ledger_hash=block['ledger_hash'], previous_txlist_hash=previous_txlist_hash, txlist_hash=block['txlist_hash'], previous_messages_hash=previous_messages_hash) logger.info('Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' % ( block['block_index'], previous_ledger_hash[-5:], previous_txlist_hash[-5:], previous_messages_hash[-5:], (' [overwrote %s]' % previous_found_messages_hash) if previous_found_messages_hash and previous_found_messages_hash != previous_messages_hash else '')) except check.ConsensusError as e: message = str(e) if message.find('ledger_hash') != -1: new_ledger = get_block_ledger(memory_db, block['block_index']) old_ledger = get_block_ledger(prod_db, block['block_index']) compare_strings(old_ledger, new_ledger) elif message.find('txlist_hash') != -1: new_txlist = get_block_txlist(memory_db, block['block_index']) old_txlist = get_block_txlist(prod_db, block['block_index']) compare_strings(old_txlist, new_txlist) raise(e)
def run(self): logger.info('Starting API Server.') self.db = self.db or database.get_connection(read_only=True, integrity_check=False) app = flask.Flask(__name__) auth = HTTPBasicAuth() @auth.get_password def get_pw(username): if username == config.RPC_USER: return config.RPC_PASSWORD return None ###################### #READ API # Generate dynamically get_{table} methods def generate_get_method(table): def get_method(**kwargs): try: return get_rows(self.db, table=table, **kwargs) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) return get_method for table in API_TABLES: new_method = generate_get_method(table) new_method.__name__ = 'get_{}'.format(table) dispatcher.add_method(new_method) @dispatcher.add_method def sql(query, bindings=None): if bindings == None: bindings = [] return db_query(self.db, query, tuple(bindings)) ###################### #WRITE/ACTION API # Generate dynamically create_{transaction} methods def generate_create_method(tx): def split_params(**kwargs): transaction_args = {} common_args = {} private_key_wif = None for key in kwargs: if key in COMMONS_ARGS: common_args[key] = kwargs[key] elif key == 'privkey': private_key_wif = kwargs[key] else: transaction_args[key] = kwargs[key] return transaction_args, common_args, private_key_wif def create_method(**kwargs): try: transaction_args, common_args, private_key_wif = split_params( **kwargs) return compose_transaction(self.db, name=tx, params=transaction_args, **common_args) except (TypeError, script.AddressError, exceptions.ComposeError, exceptions.TransactionError, exceptions.BalanceError) as error: # TypeError happens when unexpected keyword arguments are passed in error_msg = "Error composing {} transaction via API: {}".format( tx, str(error)) logging.warning(error_msg) logging.warning(traceback.format_exc()) raise JSONRPCDispatchException( code=JSON_RPC_ERROR_API_COMPOSE, message=error_msg) return create_method for tx in API_TRANSACTIONS: create_method = generate_create_method(tx) create_method.__name__ = 'create_{}'.format(tx) dispatcher.add_method(create_method) @dispatcher.add_method def get_messages(block_index): if not isinstance(block_index, int): raise APIError("block_index must be an integer.") cursor = self.db.cursor() cursor.execute( 'select * from messages where block_index = ? order by message_index asc', (block_index, )) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_messages_by_index(message_indexes): """Get specific messages from the feed, based on the message_index. @param message_index: A single index, or a list of one or more message indexes to retrieve. """ if not isinstance(message_indexes, list): message_indexes = [ message_indexes, ] for idx in message_indexes: #make sure the data is clean if not isinstance(idx, int): raise APIError( "All items in message_indexes are not integers") cursor = self.db.cursor() cursor.execute( 'SELECT * FROM messages WHERE message_index IN (%s) ORDER BY message_index ASC' % (','.join([str(x) for x in message_indexes]), )) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_supply(asset): if asset == config.BTC: return backend.get_btc_supply(normalize=False) elif asset == config.XCP: return util.xcp_supply(self.db) else: asset = util.resolve_subasset_longname(self.db, asset) return util.asset_supply(self.db, asset) @dispatcher.add_method def get_xcp_supply(): logger.warning("Deprecated method: `get_xcp_supply`") return util.xcp_supply(self.db) @dispatcher.add_method def get_asset_info(assets): logger.warning("Deprecated method: `get_asset_info`") if not isinstance(assets, list): raise APIError( "assets must be a list of asset names, even if it just contains one entry" ) assetsInfo = [] for asset in assets: asset = util.resolve_subasset_longname(self.db, asset) # BTC and XCP. if asset in [config.BTC, config.XCP]: if asset == config.BTC: supply = backend.get_btc_supply(normalize=False) else: supply = util.xcp_supply(self.db) assetsInfo.append({ 'asset': asset, 'asset_longname': None, 'owner': None, 'divisible': True, 'listed': True, 'reassignable': True, 'locked': False, 'supply': supply, 'description': '', 'issuer': None }) continue # User‐created asset. cursor = self.db.cursor() issuances = list( cursor.execute( '''SELECT * FROM issuances WHERE (status = ? AND asset = ?) ORDER BY block_index ASC''', ('valid', asset))) cursor.close() if not issuances: continue #asset not found, most likely else: last_issuance = issuances[-1] locked = False for e in issuances: if e['locked']: locked = True assetsInfo.append({ 'asset': asset, 'asset_longname': last_issuance['asset_longname'], 'owner': last_issuance['issuer'], 'divisible': bool(last_issuance['divisible']), 'listed': bool(last_issuance['listed']), 'reassignable': bool(last_issuance['reassignable']), 'locked': locked, 'supply': util.asset_supply(self.db, asset), 'description': last_issuance['description'], 'issuer': last_issuance['issuer'] }) return assetsInfo @dispatcher.add_method def get_block_info(block_index): assert isinstance(block_index, int) cursor = self.db.cursor() cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index, )) blocks = list(cursor) if len(blocks) == 1: block = blocks[0] elif len(blocks) == 0: raise exceptions.DatabaseError('No blocks found.') else: assert False cursor.close() return block @dispatcher.add_method def fee_per_kb(conf_target=config.ESTIMATE_FEE_CONF_TARGET, mode=config.ESTIMATE_FEE_MODE): return backend.fee_per_kb(conf_target, mode) @dispatcher.add_method def get_blocks(block_indexes, min_message_index=None): """fetches block info and messages for the specified block indexes @param min_message_index: Retrieve blocks from the message feed on or after this specific message index (useful since blocks may appear in the message feed more than once, if a reorg occurred). Note that if this parameter is not specified, the messages for the first block will be returned. """ if not isinstance(block_indexes, (list, tuple)): raise APIError("block_indexes must be a list of integers.") if len(block_indexes) >= 250: raise APIError("can only specify up to 250 indexes at a time.") block_indexes_str = ','.join([str(x) for x in block_indexes]) cursor = self.db.cursor() # The blocks table gets rolled back from undolog, so min_message_index doesn't matter for this query cursor.execute( 'SELECT * FROM blocks WHERE block_index IN (%s) ORDER BY block_index ASC' % (block_indexes_str, )) blocks = cursor.fetchall() cursor.execute( 'SELECT * FROM messages WHERE block_index IN (%s) ORDER BY message_index ASC' % (block_indexes_str, )) messages = collections.deque(cursor.fetchall()) # Discard any messages less than min_message_index if min_message_index: while len(messages) and messages[0][ 'message_index'] < min_message_index: messages.popleft() # Packages messages into their appropriate block in the data structure to be returned for block in blocks: block['_messages'] = [] while len(messages) and messages[0]['block_index'] == block[ 'block_index']: block['_messages'].append(messages.popleft()) #NOTE: if len(messages), then we're only returning the messages for the first set of blocks before the reorg cursor.close() return blocks @dispatcher.add_method def get_running_info(): latestBlockIndex = backend.getblockcount() try: check_database_state(self.db, latestBlockIndex) except DatabaseError: caught_up = False else: caught_up = True try: cursor = self.db.cursor() blocks = list( cursor.execute( '''SELECT * FROM blocks WHERE block_index = ?''', (util.CURRENT_BLOCK_INDEX, ))) assert len(blocks) == 1 last_block = blocks[0] cursor.close() except: last_block = None try: last_message = util.last_message(self.db) except: last_message = None try: indexd_blocks_behind = backend.getindexblocksbehind() except: indexd_blocks_behind = latestBlockIndex if latestBlockIndex > 0 else 999999 indexd_caught_up = indexd_blocks_behind <= 1 server_ready = caught_up and indexd_caught_up return { 'server_ready': server_ready, 'db_caught_up': caught_up, 'bitcoin_block_count': latestBlockIndex, 'last_block': last_block, 'indexd_caught_up': indexd_caught_up, 'indexd_blocks_behind': indexd_blocks_behind, 'last_message_index': last_message['message_index'] if last_message else -1, 'api_limit_rows': config.API_LIMIT_ROWS, 'running_testnet': config.TESTNET, 'running_regtest': config.REGTEST, 'running_testcoin': config.TESTCOIN, 'version_major': config.VERSION_MAJOR, 'version_minor': config.VERSION_MINOR, 'version_revision': config.VERSION_REVISION } @dispatcher.add_method def get_element_counts(): counts = {} cursor = self.db.cursor() for element in [ 'transactions', 'blocks', 'debits', 'credits', 'balances', 'sends', 'orders', 'order_matches', 'btcpays', 'issuances', 'broadcasts', 'bets', 'bet_matches', 'dividends', 'burns', 'cancels', 'order_expirations', 'bet_expirations', 'order_match_expirations', 'bet_match_expirations', 'messages', 'destructions' ]: cursor.execute("SELECT COUNT(*) AS count FROM %s" % element) count_list = cursor.fetchall() assert len(count_list) == 1 counts[element] = count_list[0]['count'] cursor.close() return counts @dispatcher.add_method def get_asset_names(longnames=False): cursor = self.db.cursor() if longnames: names = [] for row in cursor.execute( "SELECT asset, asset_longname FROM issuances WHERE status = 'valid' GROUP BY asset ORDER BY asset ASC" ): names.append({ 'asset': row['asset'], 'asset_longname': row['asset_longname'] }) else: names = [ row['asset'] for row in cursor.execute( "SELECT DISTINCT asset FROM issuances WHERE status = 'valid' ORDER BY asset ASC" ) ] cursor.close() return names @dispatcher.add_method def get_asset_longnames(): return get_asset_names(longnames=True) @dispatcher.add_method def get_holder_count(asset): asset = util.resolve_subasset_longname(self.db, asset) holders = util.holders(self.db, asset, True) addresses = [] for holder in holders: addresses.append(holder['address']) return {asset: len(set(addresses))} @dispatcher.add_method def get_holders(asset): asset = util.resolve_subasset_longname(self.db, asset) holders = util.holders(self.db, asset, True) return holders @dispatcher.add_method def search_raw_transactions(address, unconfirmed=True): return backend.search_raw_transactions(address, unconfirmed=unconfirmed) @dispatcher.add_method def get_unspent_txouts(address, unconfirmed=False, unspent_tx_hash=None): return backend.get_unspent_txouts(address, unconfirmed=unconfirmed, unspent_tx_hash=unspent_tx_hash) @dispatcher.add_method def getrawtransaction(tx_hash, verbose=False, skip_missing=False): return backend.getrawtransaction(tx_hash, verbose=verbose, skip_missing=skip_missing) @dispatcher.add_method def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False): return backend.getrawtransaction_batch(txhash_list, verbose=verbose, skip_missing=skip_missing) @dispatcher.add_method def get_tx_info(tx_hex, block_index=None): # block_index mandatory for transactions before block 335000 source, destination, btc_amount, fee, data, extra = blocks.get_tx_info( tx_hex, block_index=block_index) return source, destination, btc_amount, fee, util.hexlify( data) if data else '' @dispatcher.add_method def unpack(data_hex): data = binascii.unhexlify(data_hex) message_type_id, message = message_type.unpack(data) # TODO: Enabled only for `send`. if message_type_id == send.ID: unpack_method = send.unpack elif message_type_id == enhanced_send.ID: unpack_method = enhanced_send.unpack else: raise APIError('unsupported message type') unpacked = unpack_method(self.db, message, util.CURRENT_BLOCK_INDEX) return message_type_id, unpacked @dispatcher.add_method # TODO: Rename this method. def search_pubkey(pubkeyhash, provided_pubkeys=None): return backend.pubkeyhash_to_pubkey( pubkeyhash, provided_pubkeys=provided_pubkeys) def _set_cors_headers(response): if not config.RPC_NO_ALLOW_CORS: response.headers['Access-Control-Allow-Origin'] = '*' response.headers[ 'Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS' response.headers[ 'Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization' @app.route('/healthz', methods=['GET']) def handle_healthz(): msg, code = 'Healthy', 200 try: latestBlockIndex = backend.getblockcount() check_database_state(self.db, latestBlockIndex) except DatabaseError: msg, code = 'Unhealthy', 503 return flask.Response(msg, code, mimetype='text/plain') @app.route('/', defaults={'args_path': ''}, methods=['GET', 'POST', 'OPTIONS']) @app.route('/<path:args_path>', methods=['GET', 'POST', 'OPTIONS']) # Only require authentication if RPC_PASSWORD is set. @conditional_decorator(auth.login_required, hasattr(config, 'RPC_PASSWORD')) def handle_root(args_path): """Handle all paths, decide where to forward the query.""" if args_path == '' or args_path.startswith('api/') or args_path.startswith('API/') or \ args_path.startswith('rpc/') or args_path.startswith('RPC/'): if flask.request.method == 'POST': # Need to get those here because it might not be available in this aux function. request_json = flask.request.get_data().decode('utf-8') response = handle_rpc_post(request_json) return response elif flask.request.method == 'OPTIONS': response = handle_rpc_options() return response else: error = 'Invalid method.' return flask.Response(error, 405, mimetype='application/json') elif args_path.startswith('rest/') or args_path.startswith( 'REST/'): if flask.request.method == 'GET' or flask.request.method == 'POST': # Pass the URL path without /REST/ part and Flask request object. rest_path = args_path.split('/', 1)[1] response = handle_rest(rest_path, flask.request) return response else: error = 'Invalid method.' return flask.Response(error, 405, mimetype='application/json') else: # Not found return flask.Response(None, 404, mimetype='application/json') ###################### # JSON-RPC API ###################### def handle_rpc_options(): response = flask.Response('', 204) _set_cors_headers(response) return response def handle_rpc_post(request_json): """Handle /API/ POST route. Call relevant get_rows/create_transaction wrapper.""" # Check for valid request format. try: request_data = json.loads(request_json) assert 'id' in request_data and request_data[ 'jsonrpc'] == "2.0" and request_data['method'] # params may be omitted except: obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest( data="Invalid JSON-RPC 2.0 request format") return flask.Response(obj_error.json.encode(), 400, mimetype='application/json') # Only arguments passed as a `dict` are supported. if request_data.get('params', None) and not isinstance( request_data['params'], dict): obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest( data= 'Arguments must be passed as a JSON object (list of unnamed arguments not supported)' ) return flask.Response(obj_error.json.encode(), 400, mimetype='application/json') # Return an error if the API Status Poller checks fail. if not config.FORCE and current_api_status_code: return flask.Response(current_api_status_response_json, 503, mimetype='application/json') # Answer request normally. # NOTE: `UnboundLocalError: local variable 'output' referenced before assignment` means the method doesn’t return anything. jsonrpc_response = jsonrpc.JSONRPCResponseManager.handle( request_json, dispatcher) response = flask.Response(jsonrpc_response.json.encode(), 200, mimetype='application/json') _set_cors_headers(response) return response ###################### # HTTP REST API ###################### def handle_rest(path_args, flask_request): """Handle /REST/ route. Query the database using get_rows or create transaction using compose_transaction.""" url_action = flask_request.path.split('/')[-1] if url_action == 'compose': compose = True elif url_action == 'get': compose = False else: error = 'Invalid action "%s".' % url_action return flask.Response(error, 400, mimetype='application/json') # Get all arguments passed via URL. url_args = path_args.split('/') try: query_type = url_args.pop(0).lower() except IndexError: error = 'No query_type provided.' return flask.Response(error, 400, mimetype='application/json') # Check if message type or table name are valid. if (compose and query_type not in API_TRANSACTIONS) or \ (not compose and query_type not in API_TABLES): error = 'No such query type in supported queries: "%s".' % query_type return flask.Response(error, 400, mimetype='application/json') # Parse the additional arguments. extra_args = flask_request.args.items() query_data = {} if compose: common_args = {} transaction_args = {} for (key, value) in extra_args: # Determine value type. try: value = int(value) except ValueError: try: value = float(value) except ValueError: pass # Split keys into common and transaction-specific arguments. Discard the privkey. if key in COMMONS_ARGS: common_args[key] = value elif key == 'privkey': pass else: transaction_args[key] = value # Must have some additional transaction arguments. if not len(transaction_args): error = 'No transaction arguments provided.' return flask.Response(error, 400, mimetype='application/json') # Compose the transaction. try: query_data = compose_transaction(self.db, name=query_type, params=transaction_args, **common_args) except (script.AddressError, exceptions.ComposeError, exceptions.TransactionError, exceptions.BalanceError) as error: error_msg = logging.warning( "{} -- error composing {} transaction via API: {}". format(str(error.__class__.__name__), query_type, str(error))) return flask.Response(error_msg, 400, mimetype='application/json') else: # Need to de-generate extra_args to pass it through. query_args = dict([item for item in extra_args]) operator = query_args.pop('op', 'AND') # Put the data into specific dictionary format. data_filter = [{ 'field': key, 'op': '==', 'value': value } for (key, value) in query_args.items()] # Run the query. try: query_data = get_rows(self.db, table=query_type, filters=data_filter, filterop=operator) except APIError as error: return flask.Response(str(error), 400, mimetype='application/json') # See which encoding to choose from. file_format = flask_request.headers['Accept'] # JSON as default. if file_format == 'application/json' or file_format == '*/*': response_data = json.dumps(query_data) elif file_format == 'application/xml': # Add document root for XML. Note when xmltodict encounters a list, it produces separate tags for every item. # Hence we end up with multiple query_type roots. To combat this we put it in a separate item dict. response_data = serialize_to_xml( {query_type: { 'item': query_data }}) else: error = 'Invalid file format: "%s".' % file_format return flask.Response(error, 400, mimetype='application/json') response = flask.Response(response_data, 200, mimetype=file_format) return response # Init the HTTP Server. init_api_access_log(app) # Run app server (blocking) self.is_ready = True app.run(host=config.RPC_HOST, port=config.RPC_PORT, threaded=True) self.db.close() return
def reparse(testnet=True): """Reparse all transaction from the database, create a new blockchain and compare it to the old one.""" options = dict(COUNTERPARTYD_OPTIONS) server.initialise(database_file=':memory:', testnet=testnet, **options) logger = logging.getLogger() if testnet: config.PREFIX = b'TESTXXXX' memory_db = database.get_connection(read_only=False) initialise_db(memory_db) data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) prod_db_path = os.path.join( data_dir, '{}{}.db'.format(config.APP_NAME, '.testnet' if testnet else '')) assert os.path.exists( prod_db_path), "database path {} does not exist".format(prod_db_path) prod_db = apsw.Connection(prod_db_path) prod_db.setrowtrace(database.rowtracer) with memory_db.backup("main", prod_db, "main") as backup: backup.step() # Here we don’t use block.reparse() because it reparse db in transaction (`with db`). memory_cursor = memory_db.cursor() for table in blocks.TABLES + ['balances']: memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table)) # Check that all checkpoint blocks are in the database to be tested. if testnet: CHECKPOINTS = check.CHECKPOINTS_TESTNET else: CHECKPOINTS = check.CHECKPOINTS_MAINNET for block_index in CHECKPOINTS.keys(): block_exists = bool( list( memory_cursor.execute( '''SELECT * FROM blocks WHERE block_index = ?''', (block_index, )))) assert block_exists, "block #%d does not exist" % block_index # Clean consensus hashes if first block hash don’t match with checkpoint. checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET columns = [ column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''') ] for field in ['ledger_hash', 'txlist_hash']: if field in columns: sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format( field) first_hash = list( memory_cursor.execute(sql, (config.BLOCK_FIRST, )))[0][field] if first_hash != checkpoints[config.BLOCK_FIRST][field]: logger.info('First hash changed. Cleaning {}.'.format(field)) memory_cursor.execute( '''UPDATE blocks SET {} = NULL'''.format(field)) blocks.initialise(memory_db) previous_ledger_hash = None previous_txlist_hash = None previous_messages_hash = None memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''') for block in memory_cursor.fetchall(): try: util.CURRENT_BLOCK_INDEX = block['block_index'] previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block( memory_db, block['block_index'], block['block_time'], previous_ledger_hash=previous_ledger_hash, ledger_hash=block['ledger_hash'], previous_txlist_hash=previous_txlist_hash, txlist_hash=block['txlist_hash'], previous_messages_hash=previous_messages_hash) logger.info( 'Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' % (block['block_index'], previous_ledger_hash[-5:], previous_txlist_hash[-5:], previous_messages_hash[-5:], (' [overwrote %s]' % previous_found_messages_hash) if previous_found_messages_hash and previous_found_messages_hash != previous_messages_hash else '')) except check.ConsensusError as e: message = str(e) if message.find('ledger_hash') != -1: new_ledger = get_block_ledger(memory_db, block['block_index']) old_ledger = get_block_ledger(prod_db, block['block_index']) compare_strings(old_ledger, new_ledger) elif message.find('txlist_hash') != -1: new_txlist = get_block_txlist(memory_db, block['block_index']) old_txlist = get_block_txlist(prod_db, block['block_index']) compare_strings(old_txlist, new_txlist) raise (e)
def initialise(database_file=None, log_file=None, api_log_file=None, testnet=False, testcoin=False, backend_name=None, backend_connect=None, backend_port=None, backend_user=None, backend_password=None, backend_ssl=False, backend_ssl_verify=True, backend_poll_interval=None, rpc_host=None, rpc_port=None, rpc_user=None, rpc_password=None, rpc_allow_cors=None, force=False, verbose=False): # Data directory data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) if not os.path.isdir(data_dir): os.makedirs(data_dir, mode=0o755) # testnet if testnet: config.TESTNET = testnet else: config.TESTNET = False # testcoin if testcoin: config.TESTCOIN = testcoin else: config.TESTCOIN = False network = '' if config.TESTNET: network += '.testnet' if config.TESTCOIN: network += '.testcoin' # Database if database_file: config.DATABASE = database_file else: filename = '{}{}.db'.format(config.APP_NAME, network) config.DATABASE = os.path.join(data_dir, filename) # Log directory log_dir = appdirs.user_log_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME) if not os.path.isdir(log_dir): os.makedirs(log_dir, mode=0o755) # Log if log_file: config.LOG = log_file else: filename = 'server{}.log'.format(network) config.LOG = os.path.join(log_dir, filename) logger.debug('Writing server log to file: `{}`'.format(config.LOG)) if api_log_file: config.API_LOG = api_log_file else: filename = 'server{}.api.log'.format(network) config.API_LOG = os.path.join(log_dir, filename) logger.debug('Writing API log to file: `{}`'.format(config.API_LOG)) ############## # THINGS WE CONNECT TO # Backend name if backend_name: config.BACKEND_NAME = backend_name else: config.BACKEND_NAME = 'addrindex' if config.BACKEND_NAME == 'jmcorgan': config.BACKEND_NAME = 'addrindex' # Backend RPC host (Bitcoin Core) if backend_connect: config.BACKEND_CONNECT = backend_connect else: config.BACKEND_CONNECT = 'localhost' # Backend Core RPC port (Bitcoin Core) if backend_port: config.BACKEND_PORT = backend_port else: if config.TESTNET: config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT_TESTNET else: config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT try: config.BACKEND_PORT = int(config.BACKEND_PORT) if not (int(config.BACKEND_PORT) > 1 and int(config.BACKEND_PORT) < 65535): raise ConfigurationError('invalid backend API port number') except: raise ConfigurationError( "Please specific a valid port number backend-port configuration parameter" ) # Backend Core RPC user (Bitcoin Core) if backend_user: config.BACKEND_USER = backend_user else: config.BACKEND_USER = '******' # Backend Core RPC password (Bitcoin Core) if backend_password: config.BACKEND_PASSWORD = backend_password else: raise ConfigurationError( 'backend RPC password not set. (Use configuration file or --backend-password=PASSWORD)' ) # Backend Core RPC SSL if backend_ssl: config.BACKEND_SSL = backend_ssl else: config.BACKEND_SSL = False # Default to off. # Backend Core RPC SSL Verify if backend_ssl_verify is not None: config.BACKEND_SSL_VERIFY = backend_ssl_verify else: config.BACKEND_SSL_VERIFY = True # Default to on (don't support self‐signed certificates) # Backend Poll Interval if backend_poll_interval: config.BACKEND_POLL_INTERVAL = backend_poll_interval else: config.BACKEND_POLL_INTERVAL = 2.0 # Construct backend URL. config.BACKEND_URL = config.BACKEND_USER + ':' + config.BACKEND_PASSWORD + '@' + config.BACKEND_CONNECT + ':' + str( config.BACKEND_PORT) if config.BACKEND_SSL: config.BACKEND_URL = 'https://' + config.BACKEND_URL else: config.BACKEND_URL = 'http://' + config.BACKEND_URL ############## # THINGS WE SERVE # counterpartyd API RPC host if rpc_host: config.RPC_HOST = rpc_host else: config.RPC_HOST = 'localhost' # counterpartyd API RPC port if rpc_port: config.RPC_PORT = rpc_port else: if config.TESTNET: if config.TESTCOIN: config.RPC_PORT = config.DEFAULT_RPC_PORT_TESTNET + 1 else: config.RPC_PORT = config.DEFAULT_RPC_PORT_TESTNET else: if config.TESTCOIN: config.RPC_PORT = config.DEFAULT_RPC_PORT + 1 else: config.RPC_PORT = config.DEFAULT_RPC_PORT try: config.RPC_PORT = int(config.RPC_PORT) if not (int(config.RPC_PORT) > 1 and int(config.RPC_PORT) < 65535): raise ConfigurationError('invalid counterpartyd API port number') except: raise ConfigurationError( "Please specific a valid port number rpc-port configuration parameter" ) # counterpartyd API RPC user if rpc_user: config.RPC_USER = rpc_user else: config.RPC_USER = '******' # counterpartyd API RPC password if rpc_password: config.RPC_PASSWORD = rpc_password else: raise ConfigurationError( 'RPC password not set. (Use configuration file or --rpc-password=PASSWORD)' ) config.RPC = 'http://' + urlencode(config.RPC_USER) + ':' + urlencode( config.RPC_PASSWORD) + '@' + config.RPC_HOST + ':' + str( config.RPC_PORT) # RPC CORS if rpc_allow_cors is not None: config.RPC_ALLOW_CORS = rpc_allow_cors else: config.RPC_ALLOW_CORS = True ############## # OTHER SETTINGS # skip checks if force: config.FORCE = force else: config.FORCE = False # Encoding if config.TESTCOIN: config.PREFIX = b'XX' # 2 bytes (possibly accidentally created) else: config.PREFIX = b'CNTRPRTY' # 8 bytes # (more) Testnet if config.TESTNET: config.MAGIC_BYTES = config.MAGIC_BYTES_TESTNET if config.TESTCOIN: config.ADDRESSVERSION = config.ADDRESSVERSION_TESTNET config.BLOCK_FIRST = config.BLOCK_FIRST_TESTNET_TESTCOIN config.BURN_START = config.BURN_START_TESTNET_TESTCOIN config.BURN_END = config.BURN_END_TESTNET_TESTCOIN config.UNSPENDABLE = config.UNSPENDABLE_TESTNET else: config.ADDRESSVERSION = config.ADDRESSVERSION_TESTNET config.BLOCK_FIRST = config.BLOCK_FIRST_TESTNET config.BURN_START = config.BURN_START_TESTNET config.BURN_END = config.BURN_END_TESTNET config.UNSPENDABLE = config.UNSPENDABLE_TESTNET else: config.MAGIC_BYTES = config.MAGIC_BYTES_MAINNET if config.TESTCOIN: config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET config.BLOCK_FIRST = config.BLOCK_FIRST_MAINNET_TESTCOIN config.BURN_START = config.BURN_START_MAINNET_TESTCOIN config.BURN_END = config.BURN_END_MAINNET_TESTCOIN config.UNSPENDABLE = config.UNSPENDABLE_MAINNET else: config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET config.BLOCK_FIRST = config.BLOCK_FIRST_MAINNET config.BURN_START = config.BURN_START_MAINNET config.BURN_END = config.BURN_END_MAINNET config.UNSPENDABLE = config.UNSPENDABLE_MAINNET # Set up logging. root_logger = logging.getLogger() # Get root logger. log.set_up(root_logger, verbose=verbose, logfile=config.LOG) # Log unhandled errors. def handle_exception(exc_type, exc_value, exc_traceback): logger.error("Unhandled Exception", exc_info=(exc_type, exc_value, exc_traceback)) sys.excepthook = handle_exception logger.info('Running v{} of counterparty-lib.'.format( config.VERSION_STRING)) if config.FORCE: logger.warning( 'THE OPTION `--force` IS NOT FOR USE ON PRODUCTION SYSTEMS.') # Lock if not config.FORCE: get_lock() # Database logger.info('Connecting to database.') db = database.get_connection(read_only=False) util.CURRENT_BLOCK_INDEX = blocks.last_db_index(db) return db
def reparse(testnet=True): """ Reparse all transaction from the database. - Create a new in-memory DB, copy the DB that is on-disk - Reparse DB, automatically compares consensus hashes to the original ones from the on-disk DB """ options = dict(COUNTERPARTYD_OPTIONS) server.initialise(database_file=':memory:', testnet=testnet, **options) logger = logging.getLogger() if testnet: config.PREFIX = b'TESTXXXX' memory_db = database.get_connection(read_only=False) data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) prod_db_path = os.path.join( data_dir, '{}{}.db'.format(config.APP_NAME, '.testnet' if testnet else '')) assert os.path.exists( prod_db_path), "database path {} does not exist".format(prod_db_path) prod_db = apsw.Connection(prod_db_path) prod_db.setrowtrace(database.rowtracer) # Copy DB from file on disk (should be a DB file with at least all the checkpoints) # in-memory DB shouldn't have been written to yet up until this point with memory_db.backup("main", prod_db, "main") as backup: while not backup.done: backup.step(100) # Drop most tables (except blocks, transactions, undolog) memory_cursor = memory_db.cursor() for table in blocks.TABLES + ['balances']: memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table)) # Check that all checkpoint blocks are in the database to be tested. if testnet: CHECKPOINTS = check.CHECKPOINTS_TESTNET else: CHECKPOINTS = check.CHECKPOINTS_MAINNET for block_index in CHECKPOINTS.keys(): block_exists = bool( list( memory_cursor.execute( '''SELECT * FROM blocks WHERE block_index = ?''', (block_index, )))) assert block_exists, "block #%d does not exist" % block_index # Clean consensus hashes if first block hash don’t match with checkpoint. checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET columns = [ column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''') ] for field in ['ledger_hash', 'txlist_hash']: if field in columns: sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format( field) first_hash = list( memory_cursor.execute(sql, (config.BLOCK_FIRST, )))[0][field] if first_hash != checkpoints[config.BLOCK_FIRST][field]: logger.info('First hash changed. Cleaning {}.'.format(field)) memory_cursor.execute( '''UPDATE blocks SET {} = NULL'''.format(field)) # Initialise missing tables blocks.initialise(memory_db) previous_ledger_hash = None previous_txlist_hash = None previous_messages_hash = None # Reparse each block, if ConsensusError is thrown then the difference memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''') for block in memory_cursor.fetchall(): try: util.CURRENT_BLOCK_INDEX = block['block_index'] previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block( memory_db, block['block_index'], block['block_time'], previous_ledger_hash=previous_ledger_hash, ledger_hash=block['ledger_hash'], previous_txlist_hash=previous_txlist_hash, txlist_hash=block['txlist_hash'], previous_messages_hash=previous_messages_hash) logger.info( 'Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' % (block['block_index'], previous_ledger_hash[-5:], previous_txlist_hash[-5:], previous_messages_hash[-5:], (' [overwrote %s]' % previous_found_messages_hash) if previous_found_messages_hash and previous_found_messages_hash != previous_messages_hash else '')) except check.ConsensusError as e: message = str(e) if message.find('ledger_hash') != -1: new_ledger = get_block_ledger(memory_db, block['block_index']) old_ledger = get_block_ledger(prod_db, block['block_index']) compare_strings(old_ledger, new_ledger) elif message.find('txlist_hash') != -1: new_txlist = get_block_txlist(memory_db, block['block_index']) old_txlist = get_block_txlist(prod_db, block['block_index']) compare_strings(old_txlist, new_txlist) raise e
def reparse(testnet=True): """ Reparse all transaction from the database. - Create a new in-memory DB, copy the DB that is on-disk - Reparse DB, automatically compares consensus hashes to the original ones from the on-disk DB """ options = dict(COUNTERPARTYD_OPTIONS) server.initialise(database_file=':memory:', testnet=testnet, **options) logger = logging.getLogger() if testnet: config.PREFIX = b'TESTXXXX' memory_db = database.get_connection(read_only=False) data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) prod_db_path = os.path.join(data_dir, '{}{}.db'.format(config.APP_NAME, '.testnet' if testnet else '')) assert os.path.exists(prod_db_path), "database path {} does not exist".format(prod_db_path) prod_db = apsw.Connection(prod_db_path) prod_db.setrowtrace(database.rowtracer) # Copy DB from file on disk (should be a DB file with at least all the checkpoints) # in-memory DB shouldn't have been written to yet up until this point with memory_db.backup("main", prod_db, "main") as backup: while not backup.done: backup.step(100) # Drop most tables (except blocks, transactions, undolog) memory_cursor = memory_db.cursor() for table in blocks.TABLES + ['balances']: memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table)) # Check that all checkpoint blocks are in the database to be tested. if testnet: CHECKPOINTS = check.CHECKPOINTS_TESTNET else: CHECKPOINTS = check.CHECKPOINTS_MAINNET for block_index in CHECKPOINTS.keys(): block_exists = bool(list(memory_cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,)))) assert block_exists, "block #%d does not exist" % block_index # Clean consensus hashes if first block hash don’t match with checkpoint. checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET columns = [column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')] for field in ['ledger_hash', 'txlist_hash']: if field in columns: sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(field) first_hash = list(memory_cursor.execute(sql, (config.BLOCK_FIRST,)))[0][field] if first_hash != checkpoints[config.BLOCK_FIRST][field]: logger.info('First hash changed. Cleaning {}.'.format(field)) memory_cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field)) # Initialise missing tables blocks.initialise(memory_db) previous_ledger_hash = None previous_txlist_hash = None previous_messages_hash = None # Reparse each block, if ConsensusError is thrown then the difference memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''') for block in memory_cursor.fetchall(): try: util.CURRENT_BLOCK_INDEX = block['block_index'] previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = blocks.parse_block( memory_db, block['block_index'], block['block_time'], previous_ledger_hash=previous_ledger_hash, ledger_hash=block['ledger_hash'], previous_txlist_hash=previous_txlist_hash, txlist_hash=block['txlist_hash'], previous_messages_hash=previous_messages_hash) logger.info('Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' % ( block['block_index'], previous_ledger_hash[-5:], previous_txlist_hash[-5:], previous_messages_hash[-5:], (' [overwrote %s]' % previous_found_messages_hash) if previous_found_messages_hash and previous_found_messages_hash != previous_messages_hash else '')) except check.ConsensusError as e: message = str(e) if message.find('ledger_hash') != -1: new_ledger = get_block_ledger(memory_db, block['block_index']) old_ledger = get_block_ledger(prod_db, block['block_index']) compare_strings(old_ledger, new_ledger) elif message.find('txlist_hash') != -1: new_txlist = get_block_txlist(memory_db, block['block_index']) old_txlist = get_block_txlist(prod_db, block['block_index']) compare_strings(old_txlist, new_txlist) raise e
def run(self): logger.info('Starting API Server.') db = database.get_connection(read_only=True, integrity_check=False) app = flask.Flask(__name__) auth = HTTPBasicAuth() @auth.get_password def get_pw(username): if username == config.RPC_USER: return config.RPC_PASSWORD return None ###################### #READ API # Generate dynamically get_{table} methods def generate_get_method(table): def get_method(**kwargs): try: return get_rows(db, table=table, **kwargs) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) return get_method for table in API_TABLES: new_method = generate_get_method(table) new_method.__name__ = 'get_{}'.format(table) dispatcher.add_method(new_method) @dispatcher.add_method def sql(query, bindings=None): if bindings == None: bindings = [] return db_query(db, query, tuple(bindings)) ###################### #WRITE/ACTION API # Generate dynamically create_{transaction} methods def generate_create_method(tx): def split_params(**kwargs): transaction_args = {} common_args = {} private_key_wif = None for key in kwargs: if key in COMMONS_ARGS: common_args[key] = kwargs[key] elif key == 'privkey': private_key_wif = kwargs[key] else: transaction_args[key] = kwargs[key] return transaction_args, common_args, private_key_wif def create_method(**kwargs): try: transaction_args, common_args, private_key_wif = split_params(**kwargs) return compose_transaction(db, name=tx, params=transaction_args, **common_args) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) return create_method for tx in API_TRANSACTIONS: create_method = generate_create_method(tx) create_method.__name__ = 'create_{}'.format(tx) dispatcher.add_method(create_method) @dispatcher.add_method def get_messages(block_index): if not isinstance(block_index, int): raise APIError("block_index must be an integer.") cursor = db.cursor() cursor.execute('select * from messages where block_index = ? order by message_index asc', (block_index,)) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_messages_by_index(message_indexes): """Get specific messages from the feed, based on the message_index. @param message_index: A single index, or a list of one or more message indexes to retrieve. """ if not isinstance(message_indexes, list): message_indexes = [message_indexes,] for idx in message_indexes: #make sure the data is clean if not isinstance(idx, int): raise APIError("All items in message_indexes are not integers") cursor = db.cursor() cursor.execute('SELECT * FROM messages WHERE message_index IN (%s) ORDER BY message_index ASC' % (','.join([str(x) for x in message_indexes]),)) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_supply(asset): if asset == 'BTC': return backend.get_btc_supply(normalize=False) elif asset == 'XCP': return util.xcp_supply(db) else: return util.asset_supply(db, asset) @dispatcher.add_method def get_xcp_supply(): logger.warning("Deprecated method: `get_xcp_supply`") return util.xcp_supply(db) @dispatcher.add_method def get_asset_info(assets): logger.warning("Deprecated method: `get_asset_info`") if not isinstance(assets, list): raise APIError("assets must be a list of asset names, even if it just contains one entry") assetsInfo = [] for asset in assets: # BTC and XCP. if asset in [config.BTC, config.XCP]: if asset == config.BTC: supply = backend.get_btc_supply(normalize=False) else: supply = util.xcp_supply(db) assetsInfo.append({ 'asset': asset, 'owner': None, 'divisible': True, 'locked': False, 'supply': supply, 'description': '', 'issuer': None }) continue # User‐created asset. cursor = db.cursor() issuances = list(cursor.execute('''SELECT * FROM issuances WHERE (status = ? AND asset = ?) ORDER BY block_index ASC''', ('valid', asset))) cursor.close() if not issuances: continue #asset not found, most likely else: last_issuance = issuances[-1] locked = False for e in issuances: if e['locked']: locked = True assetsInfo.append({ 'asset': asset, 'owner': last_issuance['issuer'], 'divisible': bool(last_issuance['divisible']), 'locked': locked, 'supply': util.asset_supply(db, asset), 'description': last_issuance['description'], 'issuer': last_issuance['issuer']}) return assetsInfo @dispatcher.add_method def get_block_info(block_index): assert isinstance(block_index, int) cursor = db.cursor() cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,)) blocks = list(cursor) if len(blocks) == 1: block = blocks[0] elif len(blocks) == 0: raise exceptions.DatabaseError('No blocks found.') else: assert False cursor.close() return block @dispatcher.add_method def get_blocks(block_indexes): """fetches block info and messages for the specified block indexes""" if not isinstance(block_indexes, (list, tuple)): raise APIError("block_indexes must be a list of integers.") if len(block_indexes) >= 250: raise APIError("can only specify up to 250 indexes at a time.") block_indexes_str = ','.join([str(x) for x in block_indexes]) cursor = db.cursor() cursor.execute('SELECT * FROM blocks WHERE block_index IN (%s) ORDER BY block_index ASC' % (block_indexes_str,)) blocks = cursor.fetchall() cursor.execute('SELECT * FROM messages WHERE block_index IN (%s) ORDER BY block_index ASC, message_index ASC' % (block_indexes_str,)) messages = collections.deque(cursor.fetchall()) for block in blocks: # messages_in_block = [] block['_messages'] = [] while len(messages) and messages[0]['block_index'] == block['block_index']: block['_messages'].append(messages.popleft()) assert not len(messages) #should have been cleared out cursor.close() return blocks @dispatcher.add_method def get_running_info(): latestBlockIndex = backend.getblockcount() try: check_database_state(db, latestBlockIndex) except DatabaseError: caught_up = False else: caught_up = True try: cursor = db.cursor() blocks = list(cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (util.CURRENT_BLOCK_INDEX, ))) assert len(blocks) == 1 last_block = blocks[0] cursor.close() except: last_block = None try: last_message = util.last_message(db) except: last_message = None return { 'db_caught_up': caught_up, 'bitcoin_block_count': latestBlockIndex, 'last_block': last_block, 'last_message_index': last_message['message_index'] if last_message else -1, 'running_testnet': config.TESTNET, 'running_testcoin': config.TESTCOIN, 'version_major': config.VERSION_MAJOR, 'version_minor': config.VERSION_MINOR, 'version_revision': config.VERSION_REVISION } @dispatcher.add_method def get_element_counts(): counts = {} cursor = db.cursor() for element in ['transactions', 'blocks', 'debits', 'credits', 'balances', 'sends', 'orders', 'order_matches', 'btcpays', 'issuances', 'broadcasts', 'bets', 'bet_matches', 'dividends', 'burns', 'cancels', 'order_expirations', 'bet_expirations', 'order_match_expirations', 'bet_match_expirations', 'messages']: cursor.execute("SELECT COUNT(*) AS count FROM %s" % element) count_list = cursor.fetchall() assert len(count_list) == 1 counts[element] = count_list[0]['count'] cursor.close() return counts @dispatcher.add_method def get_asset_names(): cursor = db.cursor() names = [row['asset'] for row in cursor.execute("SELECT DISTINCT asset FROM issuances WHERE status = 'valid' ORDER BY asset ASC")] cursor.close() return names @dispatcher.add_method def get_holder_count(asset): holders = util.holders(db, asset) addresses = [] for holder in holders: addresses.append(holder['address']) return {asset: len(set(addresses))} @dispatcher.add_method def get_holders(asset): holders = util.holders(db, asset) return holders @dispatcher.add_method def search_raw_transactions(address, unconfirmed=True): return backend.searchrawtransactions(address, unconfirmed=unconfirmed) @dispatcher.add_method def get_unspent_txouts(address, unconfirmed=False): return backend.get_unspent_txouts(address, unconfirmed=unconfirmed, multisig_inputs=False) @dispatcher.add_method def get_tx_info(tx_hex, block_index=None): # block_index mandatory for transactions before block 335000 source, destination, btc_amount, fee, data = blocks.get_tx_info(tx_hex, block_index=block_index) return source, destination, btc_amount, fee, util.hexlify(data) if data else '' @dispatcher.add_method def unpack(data_hex): data = binascii.unhexlify(data_hex) message_type_id = struct.unpack(config.TXTYPE_FORMAT, data[:4])[0] message = data[4:] # TODO: Enabled only for `send`. if message_type_id == send.ID: unpack_method = send.unpack else: raise APIError('unsupported message type') unpacked = unpack_method(db, message, util.CURRENT_BLOCK_INDEX) return message_type_id, unpacked @dispatcher.add_method # TODO: Rename this method. def search_pubkey(pubkeyhash, provided_pubkeys=None): return backend.pubkeyhash_to_pubkey(pubkeyhash, provided_pubkeys=provided_pubkeys) def _set_cors_headers(response): if not config.RPC_NO_ALLOW_CORS: response.headers['Access-Control-Allow-Origin'] = '*' response.headers['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS' response.headers['Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type' @app.route('/', defaults={'args_path': ''}, methods=['GET', 'POST', 'OPTIONS']) @app.route('/<path:args_path>', methods=['GET', 'POST', 'OPTIONS']) # Only require authentication if RPC_PASSWORD is set. @conditional_decorator(auth.login_required, hasattr(config, 'RPC_PASSWORD')) def handle_root(args_path): """Handle all paths, decide where to forward the query.""" if args_path == '' or args_path.startswith('api/') or args_path.startswith('API/') or \ args_path.startswith('rpc/') or args_path.startswith('RPC/'): if flask.request.method == 'POST': # Need to get those here because it might not be available in this aux function. request_json = flask.request.get_data().decode('utf-8') response = handle_rpc_post(request_json) return response elif flask.request.method == 'OPTIONS': response = handle_rpc_options() return response else: error = 'Invalid method.' return flask.Response(error, 405, mimetype='application/json') elif args_path.startswith('rest/') or args_path.startswith('REST/'): if flask.request.method == 'GET' or flask.request.method == 'POST': # Pass the URL path without /REST/ part and Flask request object. rest_path = args_path.split('/', 1)[1] response = handle_rest(rest_path, flask.request) return response else: error = 'Invalid method.' return flask.Response(error, 405, mimetype='application/json') else: # Not found return flask.Response(None, 404, mimetype='application/json') ###################### # JSON-RPC API ###################### def handle_rpc_options(): response = flask.Response('', 204) _set_cors_headers(response) return response def handle_rpc_post(request_json): """Handle /API/ POST route. Call relevant get_rows/create_transaction wrapper.""" # Check for valid request format. try: request_data = json.loads(request_json) assert 'id' in request_data and request_data['jsonrpc'] == "2.0" and request_data['method'] # params may be omitted except: obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest(data="Invalid JSON-RPC 2.0 request format") return flask.Response(obj_error.json.encode(), 400, mimetype='application/json') # Only arguments passed as a `dict` are supported. if request_data.get('params', None) and not isinstance(request_data['params'], dict): obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest( data='Arguments must be passed as a JSON object (list of unnamed arguments not supported)') return flask.Response(obj_error.json.encode(), 400, mimetype='application/json') # Return an error if the API Status Poller checks fail. if not config.FORCE and current_api_status_code: return flask.Response(current_api_status_response_json, 503, mimetype='application/json') # Answer request normally. # NOTE: `UnboundLocalError: local variable 'output' referenced before assignment` means the method doesn’t return anything. jsonrpc_response = jsonrpc.JSONRPCResponseManager.handle(request_json, dispatcher) response = flask.Response(jsonrpc_response.json.encode(), 200, mimetype='application/json') _set_cors_headers(response) return response ###################### # HTTP REST API ###################### def handle_rest(path_args, flask_request): """Handle /REST/ route. Query the database using get_rows or create transaction using compose_transaction.""" url_action = flask_request.path.split('/')[-1] if url_action == 'compose': compose = True elif url_action == 'get': compose = False else: error = 'Invalid action "%s".' % url_action return flask.Response(error, 400, mimetype='application/json') # Get all arguments passed via URL. url_args = path_args.split('/') try: query_type = url_args.pop(0).lower() except IndexError: error = 'No query_type provided.' return flask.Response(error, 400, mimetype='application/json') # Check if message type or table name are valid. if (compose and query_type not in API_TRANSACTIONS) or \ (not compose and query_type not in API_TABLES): error = 'No such query type in supported queries: "%s".' % query_type return flask.Response(error, 400, mimetype='application/json') # Parse the additional arguments. extra_args = flask_request.args.items() query_data = {} if compose: common_args = {} transaction_args = {} for (key, value) in extra_args: # Determine value type. try: value = int(value) except ValueError: try: value = float(value) except ValueError: pass # Split keys into common and transaction-specific arguments. Discard the privkey. if key in COMMONS_ARGS: common_args[key] = value elif key == 'privkey': pass else: transaction_args[key] = value # Must have some additional transaction arguments. if not len(transaction_args): error = 'No transaction arguments provided.' return flask.Response(error, 400, mimetype='application/json') # Compose the transaction. try: query_data = compose_transaction(db, name=query_type, params=transaction_args, **common_args) except (script.AddressError, exceptions.ComposeError, exceptions.TransactionError, exceptions.BalanceError) as error: error_msg = str(error.__class__.__name__) + ': ' + str(error) return flask.Response(error_msg, 400, mimetype='application/json') else: # Need to de-generate extra_args to pass it through. query_args = dict([item for item in extra_args]) operator = query_args.pop('op', 'AND') # Put the data into specific dictionary format. data_filter = [{'field': key, 'op': '==', 'value': value} for (key, value) in query_args.items()] # Run the query. try: query_data = get_rows(db, table=query_type, filters=data_filter, filterop=operator) except APIError as error: return flask.Response(str(error), 400, mimetype='application/json') # See which encoding to choose from. file_format = flask_request.headers['Accept'] # JSON as default. if file_format == 'application/json' or file_format == '*/*': response_data = json.dumps(query_data) elif file_format == 'application/xml': # Add document root for XML. Note when xmltodict encounters a list, it produces separate tags for every item. # Hence we end up with multiple query_type roots. To combat this we put it in a separate item dict. response_data = serialize_to_xml({query_type: {'item': query_data}}) else: error = 'Invalid file format: "%s".' % file_format return flask.Response(error, 400, mimetype='application/json') response = flask.Response(response_data, 200, mimetype=file_format) return response # Init the HTTP Server. init_api_access_log() http_server = HTTPServer(WSGIContainer(app), xheaders=True) try: http_server.listen(config.RPC_PORT, address=config.RPC_HOST) self.is_ready = True self.ioloop.start() except OSError: raise APIError("Cannot start the API subsystem. Is server already running, or is something else listening on port {}?".format(config.RPC_PORT)) db.close() http_server.stop() self.ioloop.close() return