def run(self): logger.debug('Starting API Status Poller.') global current_api_status_code, current_api_status_response_json db = database.get_connection(integrity_check=False) while self.stop_event.is_set() != True: try: # Check version. if time.time() - self.last_version_check >= 60 * 60: # One hour since last check. logger.debug('Checking version.') code = 10 check.version(util.last_block(db)['block_index']) self.last_version_check = time.time() # Check that bitcoind is running, communicable, and caught up with the blockchain. # Check that the database has caught up with bitcoind. if time.time() - self.last_database_check > 10 * 60: # Ten minutes since last check. code = 11 logger.debug('Checking backend state.') check.backend_state(self.proxy) code = 12 logger.debug('Checking database state.') check.database_state(db, backend.getblockcount(self.proxy)) self.last_database_check = time.time() except (check.VersionError, check.BackendError, exceptions.DatabaseError) as e: exception_name = e.__class__.__name__ exception_text = str(e) logger.debug("API Status Poller: %s", exception_text) jsonrpc_response = jsonrpc.exceptions.JSONRPCServerError(message=exception_name, data=exception_text) current_api_status_code = code current_api_status_response_json = jsonrpc_response.json.encode() else: current_api_status_code = None current_api_status_response_json = None time.sleep(config.BACKEND_POLL_INTERVAL)
def setup_function(function): counterpartyd.set_options(rpc_port=9999, data_dir=tempfile.gettempdir(), database_file=tempfile.gettempdir()+'/counterpartyd.unittest.db', rpc_password='******', backend_rpc_password='******', testnet=True, testcoin=False, backend_rpc_ssl_verify=False) try: os.remove(config.DATABASE) except: pass # Connect to database. global db db = database.get_connection(read_only=False, foreign_keys=False) from lib import blocks blocks.initialise(db)
def run_scenario(scenario, rawtransactions_db): counterpartyd.set_options(database_file=':memory:', testnet=True, **COUNTERPARTYD_OPTIONS) config.PREFIX = b'TESTXXXX' util.FIRST_MULTISIG_BLOCK_TESTNET = 1 checkpoints = dict(check.CHECKPOINTS_TESTNET) check.CHECKPOINTS_TESTNET = {} logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger_buff = io.StringIO() handler = logging.StreamHandler(logger_buff) handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) logger.addHandler(handler) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) asyncio_log = logging.getLogger('asyncio') asyncio_log.setLevel(logging.ERROR) db = database.get_connection(read_only=False) initialise_db(db) raw_transactions = [] for tx in scenario: if tx[0] != 'create_next_block': module = sys.modules['lib.messages.{}'.format(tx[0])] compose = getattr(module, 'compose') unsigned_tx_hex = transaction.construct(db, get_proxy(), compose(db, *tx[1]), **tx[2]) raw_transactions.append({tx[0]: unsigned_tx_hex}) insert_raw_transaction(unsigned_tx_hex, db, rawtransactions_db) else: create_next_block(db, block_index=config.BURN_START + tx[1], parse_block=True) dump = dump_database(db) log = logger_buff.getvalue() db.close() check.CHECKPOINTS_TESTNET = checkpoints return dump, log, json.dumps(raw_transactions, indent=4)
def setup_function(function): counterpartyd.set_options(rpc_port=9999, data_dir=tempfile.gettempdir(), database_file=tempfile.gettempdir() + '/counterpartyd.unittest.db', rpc_password='******', backend_rpc_password='******', testnet=True, testcoin=False, backend_rpc_ssl_verify=False) try: os.remove(config.DATABASE) except: pass # Connect to database. global db db = database.get_connection(read_only=False, foreign_keys=False) from lib import blocks blocks.initialise(db)
def run_scenario(scenario, rawtransactions_db): counterpartyd.set_options(database_file=':memory:', testnet=True, **COUNTERPARTYD_OPTIONS) config.PREFIX = b'TESTXXXX' util.FIRST_MULTISIG_BLOCK_TESTNET = 1 checkpoints = dict(check.CHECKPOINTS_TESTNET) check.CHECKPOINTS_TESTNET = {} proxy = RpcProxy() logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger_buff = io.StringIO() handler = logging.StreamHandler(logger_buff) handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) logger.addHandler(handler) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) asyncio_log = logging.getLogger('asyncio') asyncio_log.setLevel(logging.ERROR) db = database.get_connection(read_only=False) initialise_db(db) raw_transactions = [] for transaction in scenario: if transaction[0] != 'create_next_block': module = sys.modules['lib.messages.{}'.format(transaction[0])] compose = getattr(module, 'compose') unsigned_tx_hex = bitcoin.transaction(db, compose(db, *transaction[1]), **transaction[2]) raw_transactions.append({transaction[0]: unsigned_tx_hex}) insert_raw_transaction(unsigned_tx_hex, db, rawtransactions_db) else: create_next_block(db, block_index=config.BURN_START + transaction[1], parse_block=True) dump = dump_database(db) log = logger_buff.getvalue() db.close() check.CHECKPOINTS_TESTNET = checkpoints return dump, log, json.dumps(raw_transactions, indent=4)
def run(self): logger.debug('Starting API Status Poller.') global current_api_status_code, current_api_status_response_json db = database.get_connection(integrity_check=False) while self.stop_event.is_set() != True: try: # Check version. if time.time( ) - self.last_version_check >= 60 * 60: # One hour since last check. logger.debug('Checking version.') code = 10 check.version(util.last_block(db)['block_index']) self.last_version_check = time.time() # Check that bitcoind is running, communicable, and caught up with the blockchain. # Check that the database has caught up with bitcoind. if time.time( ) - self.last_database_check > 10 * 60: # Ten minutes since last check. code = 11 logger.debug('Checking backend state.') check.backend_state(self.proxy) code = 12 logger.debug('Checking database state.') check.database_state(db, backend.getblockcount(self.proxy)) self.last_database_check = time.time() except (check.VersionError, check.BackendError, exceptions.DatabaseError) as e: exception_name = e.__class__.__name__ exception_text = str(e) logger.debug("API Status Poller: %s", exception_text) jsonrpc_response = jsonrpc.exceptions.JSONRPCServerError( message=exception_name, data=exception_text) current_api_status_code = code current_api_status_response_json = jsonrpc_response.json.encode( ) else: current_api_status_code = None current_api_status_response_json = None time.sleep(config.BACKEND_POLL_INTERVAL)
def counterpartyd_db(request): db = database.get_connection(read_only=False) cursor = db.cursor() cursor.execute('''BEGIN''') request.addfinalizer(lambda: cursor.execute('''ROLLBACK''')) return db
import settings import logging import datetime from lib import database, weatherdata from WeatherService import app # setup the logging logFormatStr = '[%(asctime)s] p%(process)s {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s' logging.basicConfig(format = logFormatStr, filename = settings.LOG_FILE, level=settings.LOG_LEVEL) formatter = logging.Formatter(logFormatStr,'%m-%d %H:%M:%S') streamHandler = logging.StreamHandler() streamHandler.setLevel(logging.DEBUG) streamHandler.setFormatter(formatter) app.logger.addHandler(streamHandler) # create database if not exists - just an open connection needed for that conn = database.get_connection() try: conn = database.get_connection() except: app.logger.error('Can\'t connect to the database.') # request data from weather webservice and stores in the db for city in settings.OPENWEATHERMAP_CITIES: resp = weatherdata.query_weather_data(city) weatherdata.load_data(resp)
def reparse(testnet=True): options = dict(COUNTERPARTYD_OPTIONS) options.pop('data_dir') counterpartyd.set_options(database_file=':memory:', testnet=testnet, **options) if testnet: config.PREFIX = b'TESTXXXX' logger = logging.getLogger() console = logging.StreamHandler() console.setLevel(logging.INFO) formatter = logging.Formatter('%(message)s') console.setFormatter(formatter) logger.addHandler(console) memory_db = database.get_connection(read_only=False) initialise_db(memory_db) prod_db_path = os.path.join(config.DATA_DIR, '{}.{}{}.db'.format(config.XCP_CLIENT, str(config.VERSION_MAJOR), '.testnet' if testnet else '')) prod_db = apsw.Connection(prod_db_path) prod_db.setrowtrace(database.rowtracer) with memory_db.backup("main", prod_db, "main") as backup: backup.step() # here we don't use block.reparse() because it reparse db in transaction (`with db`) memory_cursor = memory_db.cursor() for table in blocks.TABLES + ['balances']: memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table)) # clean consensus hashes if first block hash don't match with checkpoint. checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET columns = [column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')] for field in ['ledger_hash', 'txlist_hash']: if field in columns: sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(field) first_hash = list(memory_cursor.execute(sql, (config.BLOCK_FIRST,)))[0][field] if first_hash != checkpoints[config.BLOCK_FIRST][field]: logging.info('First hash changed. Cleaning {}.'.format(field)) memory_cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field)) blocks.initialise(memory_db) previous_ledger_hash = None previous_txlist_hash = None memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''') for block in memory_cursor.fetchall(): try: logger.info('Block (re‐parse): {}'.format(str(block['block_index']))) previous_ledger_hash, previous_txlist_hash = blocks.parse_block(memory_db, block['block_index'], block['block_time'], previous_ledger_hash, block['ledger_hash'], previous_txlist_hash, block['txlist_hash']) except check.ConsensusError as e: message = str(e) if message.find('ledger_hash') != -1: new_ledger = get_block_ledger(memory_db, block['block_index']) old_ledger = get_block_ledger(prod_db, block['block_index']) compare_strings(old_ledger, new_ledger) elif message.find('txlist_hash') != -1: new_txlist = get_block_txlist(memory_db, block['block_index']) old_txlist = get_block_txlist(prod_db, block['block_index']) compare_strings(old_txlist, new_txlist) raise(e)
def run(self): logger.debug('Starting API Server.') db = database.get_connection(integrity_check=False) app = flask.Flask(__name__) auth = HTTPBasicAuth() @auth.get_password def get_pw(username): if username == config.RPC_USER: return config.RPC_PASSWORD return None ###################### #READ API # Generate dynamically get_{table} methods def generate_get_method(table): def get_method(**kwargs): try: return get_rows(db, table=table, **kwargs) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) return get_method for table in API_TABLES: new_method = generate_get_method(table) new_method.__name__ = 'get_{}'.format(table) dispatcher.add_method(new_method) @dispatcher.add_method def sql(query, bindings=None): if bindings == None: bindings = [] return db_query(db, query, tuple(bindings)) ###################### #WRITE/ACTION API # Generate dynamically create_{transaction} and do_{transaction} methods def generate_create_method(tx): def split_params(**kwargs): transaction_args = {} common_args = {} private_key_wif = None for key in kwargs: if key in COMMONS_ARGS: common_args[key] = kwargs[key] elif key == 'privkey': private_key_wif = kwargs[key] else: transaction_args[key] = kwargs[key] return transaction_args, common_args, private_key_wif def create_method(**kwargs): try: transaction_args, common_args, private_key_wif = split_params( **kwargs) return compose_transaction(db, self.proxy, name=tx, params=transaction_args, **common_args) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) def do_method(**kwargs): try: transaction_args, common_args, private_key_wif = split_params( **kwargs) return do_transaction(db, self.proxy, name=tx, params=transaction_args, private_key_wif=private_key_wif, **common_args) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) return create_method, do_method for tx in API_TRANSACTIONS: create_method, do_method = generate_create_method(tx) create_method.__name__ = 'create_{}'.format(tx) do_method.__name__ = 'do_{}'.format(tx) dispatcher.add_method(create_method) dispatcher.add_method(do_method) @dispatcher.add_method def sign_tx(unsigned_tx_hex, privkey=None): return sign_transaction(self.proxy, unsigned_tx_hex, private_key_wif=privkey) @dispatcher.add_method def broadcast_tx(signed_tx_hex): return broadcast_transaction(self.proxy, signed_tx_hex) @dispatcher.add_method def get_messages(block_index): if not isinstance(block_index, int): raise APIError("block_index must be an integer.") cursor = db.cursor() cursor.execute( 'select * from messages where block_index = ? order by message_index asc', (block_index, )) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_messages_by_index(message_indexes): """Get specific messages from the feed, based on the message_index. @param message_index: A single index, or a list of one or more message indexes to retrieve. """ if not isinstance(message_indexes, list): message_indexes = [ message_indexes, ] for idx in message_indexes: #make sure the data is clean if not isinstance(idx, int): raise APIError( "All items in message_indexes are not integers") cursor = db.cursor() cursor.execute( 'SELECT * FROM messages WHERE message_index IN (%s) ORDER BY message_index ASC' % (','.join([str(x) for x in message_indexes]), )) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_xcp_supply(): return util.xcp_supply(db) @dispatcher.add_method def get_asset_info(assets): if not isinstance(assets, list): raise APIError( "assets must be a list of asset names, even if it just contains one entry" ) assetsInfo = [] for asset in assets: # BTC and XCP. if asset in [config.BTC, config.XCP]: if asset == config.BTC: supply = backend.get_btc_supply(self.proxy, normalize=False) else: supply = util.xcp_supply(db) assetsInfo.append({ 'asset': asset, 'owner': None, 'divisible': True, 'locked': False, 'supply': supply, 'description': '', 'issuer': None }) continue # User‐created asset. cursor = db.cursor() issuances = list( cursor.execute( '''SELECT * FROM issuances WHERE (status = ? AND asset = ?) ORDER BY block_index ASC''', ('valid', asset))) cursor.close() if not issuances: continue #asset not found, most likely else: last_issuance = issuances[-1] locked = False for e in issuances: if e['locked']: locked = True assetsInfo.append({ 'asset': asset, 'owner': last_issuance['issuer'], 'divisible': bool(last_issuance['divisible']), 'locked': locked, 'supply': util.asset_supply(db, asset), 'description': last_issuance['description'], 'issuer': last_issuance['issuer'] }) return assetsInfo @dispatcher.add_method def get_block_info(block_index): assert isinstance(block_index, int) cursor = db.cursor() cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index, )) blocks = list(cursor) if len(blocks) == 1: block = blocks[0] elif len(blocks) == 0: raise exceptions.DatabaseError('No blocks found.') else: assert False cursor.close() return block @dispatcher.add_method def get_blocks(block_indexes): """fetches block info and messages for the specified block indexes""" if not isinstance(block_indexes, (list, tuple)): raise APIError("block_indexes must be a list of integers.") if len(block_indexes) >= 250: raise APIError("can only specify up to 250 indexes at a time.") block_indexes_str = ','.join([str(x) for x in block_indexes]) cursor = db.cursor() cursor.execute( 'SELECT * FROM blocks WHERE block_index IN (%s) ORDER BY block_index ASC' % (block_indexes_str, )) blocks = cursor.fetchall() cursor.execute( 'SELECT * FROM messages WHERE block_index IN (%s) ORDER BY block_index ASC, message_index ASC' % (block_indexes_str, )) messages = collections.deque(cursor.fetchall()) for block in blocks: # messages_in_block = [] block['_messages'] = [] while len(messages) and messages[0]['block_index'] == block[ 'block_index']: block['_messages'].append(messages.popleft()) assert not len(messages) #should have been cleared out cursor.close() return blocks @dispatcher.add_method def get_running_info(): latestBlockIndex = backend.getblockcount(self.proxy) try: check.database_state(db, latestBlockIndex) except exceptions.DatabaseError: caught_up = False else: caught_up = True try: last_block = util.last_block(db) except: last_block = { 'block_index': None, 'block_hash': None, 'block_time': None } try: last_message = util.last_message(db) except: last_message = None return { 'db_caught_up': caught_up, 'bitcoin_block_count': latestBlockIndex, 'last_block': last_block, 'last_message_index': last_message['message_index'] if last_message else -1, 'running_testnet': config.TESTNET, 'running_testcoin': config.TESTCOIN, 'version_major': config.VERSION_MAJOR, 'version_minor': config.VERSION_MINOR, 'version_revision': config.VERSION_REVISION } @dispatcher.add_method def get_element_counts(): counts = {} cursor = db.cursor() for element in [ 'transactions', 'blocks', 'debits', 'credits', 'balances', 'sends', 'orders', 'order_matches', 'btcpays', 'issuances', 'broadcasts', 'bets', 'bet_matches', 'dividends', 'burns', 'cancels', 'order_expirations', 'bet_expirations', 'order_match_expirations', 'bet_match_expirations', 'messages' ]: cursor.execute("SELECT COUNT(*) AS count FROM %s" % element) count_list = cursor.fetchall() assert len(count_list) == 1 counts[element] = count_list[0]['count'] cursor.close() return counts @dispatcher.add_method def get_asset_names(): cursor = db.cursor() names = [ row['asset'] for row in cursor.execute( "SELECT DISTINCT asset FROM issuances WHERE status = 'valid' ORDER BY asset ASC" ) ] cursor.close() return names @dispatcher.add_method def get_holder_count(asset): holders = util.holders(db, asset) addresses = [] for holder in holders: addresses.append(holder['address']) return {asset: len(set(addresses))} @dispatcher.add_method def search_raw_transactions(address): return blockchain.searchrawtransactions(self.proxy, address) @dispatcher.add_method def get_unspent_txouts(address, return_confirmed=False): result = backend.get_unspent_txouts( self.proxy, address, return_confirmed=return_confirmed) if return_confirmed: return {'all': result[0], 'confirmed': result[1]} else: return result @dispatcher.add_method def get_wallet(): # TODO: Dupe with `backend.get_wallet()` wallet = {} for group in backend.listaddressgroupings(self.proxy): for bunch in group: address, btc_balance = bunch[:2] wallet[address] = str(btc_balance) return wallet @dispatcher.add_method def get_tx_info(tx_hex): source, destination, btc_amount, fee, data = blocks.get_tx_info( self.proxy, tx_hex, util.last_block(db)['block_index']) return source, destination, btc_amount, fee, util.hexlify(data) @dispatcher.add_method def unpack(data_hex): data = binascii.unhexlify(data_hex) message_type_id = struct.unpack(config.TXTYPE_FORMAT, data[:4])[0] message = data[4:] for message_type in API_TRANSACTIONS: if message_type_id == sys.modules['lib.messages.{}'.format( message_type)].ID: unpack_method = sys.modules['lib.messages.{}'.format( message_type)].unpack unpacked = unpack_method( db, message, util.last_block(db)['block_index']) return message_type_id, unpacked def _set_cors_headers(response): if config.RPC_ALLOW_CORS: response.headers['Access-Control-Allow-Origin'] = '*' response.headers[ 'Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS' response.headers[ 'Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type' @app.route('/', methods=[ "OPTIONS", ]) @app.route('/api/', methods=[ "OPTIONS", ]) def handle_options(): response = flask.Response('', 204) _set_cors_headers(response) return response @app.route('/', methods=[ "POST", ]) @app.route('/api/', methods=[ "POST", ]) @auth.login_required def handle_post(): try: request_json = flask.request.get_data().decode('utf-8') request_data = json.loads(request_json) assert 'id' in request_data and request_data[ 'jsonrpc'] == "2.0" and request_data['method'] # params may be omitted except: obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest( data="Invalid JSON-RPC 2.0 request format") return flask.Response(obj_error.json.encode(), 200, mimetype='application/json') #only arguments passed as a dict are supported if request_data.get('params', None) and not isinstance( request_data['params'], dict): obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest( data= 'Arguments must be passed as a JSON object (list of unnamed arguments not supported)' ) return flask.Response(obj_error.json.encode(), 200, mimetype='application/json') #return an error if API fails checks if not config.FORCE and current_api_status_code: return flask.Response(current_api_status_response_json, 200, mimetype='application/json') jsonrpc_response = jsonrpc.JSONRPCResponseManager.handle( request_json, dispatcher) response = flask.Response(jsonrpc_response.json.encode(), 200, mimetype='application/json') _set_cors_headers(response) return response init_api_access_log() http_server = HTTPServer(WSGIContainer(app), xheaders=True) try: http_server.listen(config.RPC_PORT, address=config.RPC_HOST) self.is_ready = True self.ioloop.start() except OSError: raise APIError( "Cannot start the API subsystem. Is {} already running, or is something else listening on port {}?" .format(config.XCP_CLIENT, config.RPC_PORT)) db.close() http_server.stop() self.ioloop.close() return
def reparse(testnet=True): options = dict(COUNTERPARTYD_OPTIONS) options.pop('data_dir') counterpartyd.set_options(database_file=':memory:', testnet=testnet, **options) if testnet: config.PREFIX = b'TESTXXXX' logger = logging.getLogger() console = logging.StreamHandler() console.setLevel(logging.INFO) formatter = logging.Formatter('%(message)s') console.setFormatter(formatter) logger.addHandler(console) memory_db = database.get_connection(read_only=False) initialise_db(memory_db) prod_db_path = os.path.join( config.DATA_DIR, '{}.{}{}.db'.format(config.XCP_CLIENT, str(config.VERSION_MAJOR), '.testnet' if testnet else '')) prod_db = apsw.Connection(prod_db_path) prod_db.setrowtrace(database.rowtracer) with memory_db.backup("main", prod_db, "main") as backup: backup.step() # here we don't use block.reparse() because it reparse db in transaction (`with db`) memory_cursor = memory_db.cursor() for table in blocks.TABLES + ['balances']: memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table)) # clean consensus hashes if first block hash don't match with checkpoint. checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET columns = [ column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''') ] for field in ['ledger_hash', 'txlist_hash']: if field in columns: sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format( field) first_hash = list( memory_cursor.execute(sql, (config.BLOCK_FIRST, )))[0][field] if first_hash != checkpoints[config.BLOCK_FIRST][field]: logger.info('First hash changed. Cleaning {}.'.format(field)) memory_cursor.execute( '''UPDATE blocks SET {} = NULL'''.format(field)) blocks.initialise(memory_db) previous_ledger_hash = None previous_txlist_hash = None memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''') for block in memory_cursor.fetchall(): try: logger.info('Block (re‐parse): {}'.format(str( block['block_index']))) previous_ledger_hash, previous_txlist_hash = blocks.parse_block( memory_db, block['block_index'], block['block_time'], previous_ledger_hash, block['ledger_hash'], previous_txlist_hash, block['txlist_hash']) except check.ConsensusError as e: message = str(e) if message.find('ledger_hash') != -1: new_ledger = get_block_ledger(memory_db, block['block_index']) old_ledger = get_block_ledger(prod_db, block['block_index']) compare_strings(old_ledger, new_ledger) elif message.find('txlist_hash') != -1: new_txlist = get_block_txlist(memory_db, block['block_index']) old_txlist = get_block_txlist(prod_db, block['block_index']) compare_strings(old_txlist, new_txlist) raise (e)
# Version if args.action in ('server', 'reparse', 'rollback') and not config.FORCE: logger.info('Checking version.') try: check.version(backend.getblockcount(proxy)) except check.VersionUpdateRequiredError as e: traceback.print_exc(file=sys.stdout) sys.exit(config.EXITCODE_UPDATE_REQUIRED) # Lock if args.action in ('rollback', 'reparse', 'server', 'kickstart') and not config.FORCE: get_lock() # Database logger.info('Connecting to database.') db = database.get_connection(read_only=False) # MESSAGE CREATION if args.action == 'send': if args.fee: args.fee = util.value_in(db, args.fee, config.BTC) quantity = util.value_in(db, args.quantity, args.asset) cli('create_send', {'source': args.source, 'destination': args.destination, 'asset': args.asset, 'quantity': quantity, 'fee': args.fee, 'allow_unconfirmed_inputs': args.unconfirmed, 'encoding': args.encoding, 'fee_per_kb': args.fee_per_kb, 'regular_dust_size': args.regular_dust_size, 'multisig_dust_size': args.multisig_dust_size, 'op_return_value': args.op_return_value},
parser.add_argument( '-v', '--verbose', action='count', default=0, help='increase verbosity') parser.add_argument( '-n', '--no-history', action='store_true', help='do not collect historical data') args = parser.parse_args() if args.verbose > 1: loglevel = logging.DEBUG elif args.verbose > 0: loglevel = logging.INFO else: loglevel = logging.WARNING logging.basicConfig(level=loglevel, format=LOGFORMAT) config = Config(args.config) cloudtrax = CloudTrax(config) cloudtrax.collect_networks() cloudtrax.collect_nodes() if not args.no_history: cloudtrax.collect_node_history() cloudtrax.collect_clients() dbconf = config.get_db() database = get_connection(dbconf['type'], **dbconf) database.store_data(cloudtrax)
def run(self): logger.debug('Starting API Server.') db = database.get_connection(integrity_check=False) app = flask.Flask(__name__) auth = HTTPBasicAuth() @auth.get_password def get_pw(username): if username == config.RPC_USER: return config.RPC_PASSWORD return None ###################### #READ API # Generate dynamically get_{table} methods def generate_get_method(table): def get_method(**kwargs): try: return get_rows(db, table=table, **kwargs) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) return get_method for table in API_TABLES: new_method = generate_get_method(table) new_method.__name__ = 'get_{}'.format(table) dispatcher.add_method(new_method) @dispatcher.add_method def sql(query, bindings=None): if bindings == None: bindings = [] return db_query(db, query, tuple(bindings)) ###################### #WRITE/ACTION API # Generate dynamically create_{transaction} and do_{transaction} methods def generate_create_method(tx): def split_params(**kwargs): transaction_args = {} common_args = {} private_key_wif = None for key in kwargs: if key in COMMONS_ARGS: common_args[key] = kwargs[key] elif key == 'privkey': private_key_wif = kwargs[key] else: transaction_args[key] = kwargs[key] return transaction_args, common_args, private_key_wif def create_method(**kwargs): try: transaction_args, common_args, private_key_wif = split_params(**kwargs) return compose_transaction(db, self.proxy, name=tx, params=transaction_args, **common_args) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) def do_method(**kwargs): try: transaction_args, common_args, private_key_wif = split_params(**kwargs) return do_transaction(db, self.proxy, name=tx, params=transaction_args, private_key_wif=private_key_wif, **common_args) except TypeError as e: #TODO: generalise for all API methods raise APIError(str(e)) return create_method, do_method for tx in API_TRANSACTIONS: create_method, do_method = generate_create_method(tx) create_method.__name__ = 'create_{}'.format(tx) do_method.__name__ = 'do_{}'.format(tx) dispatcher.add_method(create_method) dispatcher.add_method(do_method) @dispatcher.add_method def sign_tx(unsigned_tx_hex, privkey=None): return sign_transaction(self.proxy, unsigned_tx_hex, private_key_wif=privkey) @dispatcher.add_method def broadcast_tx(signed_tx_hex): return broadcast_transaction(self.proxy, signed_tx_hex) @dispatcher.add_method def get_messages(block_index): if not isinstance(block_index, int): raise APIError("block_index must be an integer.") cursor = db.cursor() cursor.execute('select * from messages where block_index = ? order by message_index asc', (block_index,)) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_messages_by_index(message_indexes): """Get specific messages from the feed, based on the message_index. @param message_index: A single index, or a list of one or more message indexes to retrieve. """ if not isinstance(message_indexes, list): message_indexes = [message_indexes,] for idx in message_indexes: #make sure the data is clean if not isinstance(idx, int): raise APIError("All items in message_indexes are not integers") cursor = db.cursor() cursor.execute('SELECT * FROM messages WHERE message_index IN (%s) ORDER BY message_index ASC' % (','.join([str(x) for x in message_indexes]),)) messages = cursor.fetchall() cursor.close() return messages @dispatcher.add_method def get_xcp_supply(): return util.xcp_supply(db) @dispatcher.add_method def get_asset_info(assets): if not isinstance(assets, list): raise APIError("assets must be a list of asset names, even if it just contains one entry") assetsInfo = [] for asset in assets: # BTC and XCP. if asset in [config.BTC, config.XCP]: if asset == config.BTC: supply = backend.get_btc_supply(self.proxy, normalize=False) else: supply = util.xcp_supply(db) assetsInfo.append({ 'asset': asset, 'owner': None, 'divisible': True, 'locked': False, 'supply': supply, 'description': '', 'issuer': None }) continue # User‐created asset. cursor = db.cursor() issuances = list(cursor.execute('''SELECT * FROM issuances WHERE (status = ? AND asset = ?) ORDER BY block_index ASC''', ('valid', asset))) cursor.close() if not issuances: continue #asset not found, most likely else: last_issuance = issuances[-1] locked = False for e in issuances: if e['locked']: locked = True assetsInfo.append({ 'asset': asset, 'owner': last_issuance['issuer'], 'divisible': bool(last_issuance['divisible']), 'locked': locked, 'supply': util.asset_supply(db, asset), 'description': last_issuance['description'], 'issuer': last_issuance['issuer']}) return assetsInfo @dispatcher.add_method def get_block_info(block_index): assert isinstance(block_index, int) cursor = db.cursor() cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,)) blocks = list(cursor) if len(blocks) == 1: block = blocks[0] elif len(blocks) == 0: raise exceptions.DatabaseError('No blocks found.') else: assert False cursor.close() return block @dispatcher.add_method def get_blocks(block_indexes): """fetches block info and messages for the specified block indexes""" if not isinstance(block_indexes, (list, tuple)): raise APIError("block_indexes must be a list of integers.") if len(block_indexes) >= 250: raise APIError("can only specify up to 250 indexes at a time.") block_indexes_str = ','.join([str(x) for x in block_indexes]) cursor = db.cursor() cursor.execute('SELECT * FROM blocks WHERE block_index IN (%s) ORDER BY block_index ASC' % (block_indexes_str,)) blocks = cursor.fetchall() cursor.execute('SELECT * FROM messages WHERE block_index IN (%s) ORDER BY block_index ASC, message_index ASC' % (block_indexes_str,)) messages = collections.deque(cursor.fetchall()) for block in blocks: # messages_in_block = [] block['_messages'] = [] while len(messages) and messages[0]['block_index'] == block['block_index']: block['_messages'].append(messages.popleft()) assert not len(messages) #should have been cleared out cursor.close() return blocks @dispatcher.add_method def get_running_info(): latestBlockIndex = backend.getblockcount(self.proxy) try: check.database_state(db, latestBlockIndex) except exceptions.DatabaseError: caught_up = False else: caught_up = True try: last_block = util.last_block(db) except: last_block = {'block_index': None, 'block_hash': None, 'block_time': None} try: last_message = util.last_message(db) except: last_message = None return { 'db_caught_up': caught_up, 'bitcoin_block_count': latestBlockIndex, 'last_block': last_block, 'last_message_index': last_message['message_index'] if last_message else -1, 'running_testnet': config.TESTNET, 'running_testcoin': config.TESTCOIN, 'version_major': config.VERSION_MAJOR, 'version_minor': config.VERSION_MINOR, 'version_revision': config.VERSION_REVISION } @dispatcher.add_method def get_element_counts(): counts = {} cursor = db.cursor() for element in ['transactions', 'blocks', 'debits', 'credits', 'balances', 'sends', 'orders', 'order_matches', 'btcpays', 'issuances', 'broadcasts', 'bets', 'bet_matches', 'dividends', 'burns', 'cancels', 'order_expirations', 'bet_expirations', 'order_match_expirations', 'bet_match_expirations', 'messages']: cursor.execute("SELECT COUNT(*) AS count FROM %s" % element) count_list = cursor.fetchall() assert len(count_list) == 1 counts[element] = count_list[0]['count'] cursor.close() return counts @dispatcher.add_method def get_asset_names(): cursor = db.cursor() names = [row['asset'] for row in cursor.execute("SELECT DISTINCT asset FROM issuances WHERE status = 'valid' ORDER BY asset ASC")] cursor.close() return names @dispatcher.add_method def get_holder_count(asset): holders = util.holders(db, asset) addresses = [] for holder in holders: addresses.append(holder['address']) return {asset: len(set(addresses))} @dispatcher.add_method def search_raw_transactions(address): return blockchain.searchrawtransactions(self.proxy, address) @dispatcher.add_method def get_unspent_txouts(address, return_confirmed=False): result = backend.get_unspent_txouts(self.proxy, address, return_confirmed=return_confirmed) if return_confirmed: return {'all': result[0], 'confirmed': result[1]} else: return result @dispatcher.add_method def get_wallet(): # TODO: Dupe with `backend.get_wallet()` wallet = {} for group in backend.listaddressgroupings(self.proxy): for bunch in group: address, btc_balance = bunch[:2] wallet[address] = str(btc_balance) return wallet @dispatcher.add_method def get_tx_info(tx_hex): source, destination, btc_amount, fee, data = blocks.get_tx_info(tx_hex, util.last_block(db)['block_index']) return source, destination, btc_amount, fee, util.hexlify(data) @dispatcher.add_method def unpack(data_hex): data = binascii.unhexlify(data_hex) message_type_id = struct.unpack(config.TXTYPE_FORMAT, data[:4])[0] message = data[4:] for message_type in API_TRANSACTIONS: if message_type_id == sys.modules['lib.messages.{}'.format(message_type)].ID: unpack_method = sys.modules['lib.messages.{}'.format(message_type)].unpack unpacked = unpack_method(db, message, util.last_block(db)['block_index']) return message_type_id, unpacked def _set_cors_headers(response): if config.RPC_ALLOW_CORS: response.headers['Access-Control-Allow-Origin'] = '*' response.headers['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS' response.headers['Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type' @app.route('/', methods=["OPTIONS",]) @app.route('/api/', methods=["OPTIONS",]) def handle_options(): response = flask.Response('', 204) _set_cors_headers(response) return response @app.route('/', methods=["POST",]) @app.route('/api/', methods=["POST",]) @auth.login_required def handle_post(): try: request_json = flask.request.get_data().decode('utf-8') request_data = json.loads(request_json) assert 'id' in request_data and request_data['jsonrpc'] == "2.0" and request_data['method'] # params may be omitted except: obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest(data="Invalid JSON-RPC 2.0 request format") return flask.Response(obj_error.json.encode(), 200, mimetype='application/json') #only arguments passed as a dict are supported if request_data.get('params', None) and not isinstance(request_data['params'], dict): obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest( data='Arguments must be passed as a JSON object (list of unnamed arguments not supported)') return flask.Response(obj_error.json.encode(), 200, mimetype='application/json') #return an error if API fails checks if not config.FORCE and current_api_status_code: return flask.Response(current_api_status_response_json, 200, mimetype='application/json') jsonrpc_response = jsonrpc.JSONRPCResponseManager.handle(request_json, dispatcher) response = flask.Response(jsonrpc_response.json.encode(), 200, mimetype='application/json') _set_cors_headers(response) return response init_api_access_log() http_server = HTTPServer(WSGIContainer(app), xheaders=True) try: http_server.listen(config.RPC_PORT, address=config.RPC_HOST) self.is_ready = True self.ioloop.start() except OSError: raise APIError("Cannot start the API subsystem. Is {} already running, or is something else listening on port {}?".format(config.XCP_CLIENT, config.RPC_PORT)) db.close() http_server.stop() self.ioloop.close() return