def run(self):
logger.debug('Starting API Status Poller.')
global current_api_status_code, current_api_status_response_json
db = database.get_connection(integrity_check=False)
while self.stop_event.is_set() != True:
try:
# Check version.
if time.time() - self.last_version_check >= 60 * 60: # One hour since last check.
logger.debug('Checking version.')
code = 10
check.version(util.last_block(db)['block_index'])
self.last_version_check = time.time()
# Check that bitcoind is running, communicable, and caught up with the blockchain.
# Check that the database has caught up with bitcoind.
if time.time() - self.last_database_check > 10 * 60: # Ten minutes since last check.
code = 11
logger.debug('Checking backend state.')
check.backend_state(self.proxy)
code = 12
logger.debug('Checking database state.')
check.database_state(db, backend.getblockcount(self.proxy))
self.last_database_check = time.time()
except (check.VersionError, check.BackendError, exceptions.DatabaseError) as e:
exception_name = e.__class__.__name__
exception_text = str(e)
logger.debug("API Status Poller: %s", exception_text)
jsonrpc_response = jsonrpc.exceptions.JSONRPCServerError(message=exception_name, data=exception_text)
current_api_status_code = code
current_api_status_response_json = jsonrpc_response.json.encode()
else:
current_api_status_code = None
current_api_status_response_json = None
time.sleep(config.BACKEND_POLL_INTERVAL)
def setup_function(function):
counterpartyd.set_options(rpc_port=9999, data_dir=tempfile.gettempdir(), database_file=tempfile.gettempdir()+'/counterpartyd.unittest.db',
rpc_password='******', backend_rpc_password='******',
testnet=True, testcoin=False, backend_rpc_ssl_verify=False)
try:
os.remove(config.DATABASE)
except:
pass
# Connect to database.
global db
db = database.get_connection(read_only=False, foreign_keys=False)
from lib import blocks
blocks.initialise(db)
def run_scenario(scenario, rawtransactions_db):
counterpartyd.set_options(database_file=':memory:',
testnet=True,
**COUNTERPARTYD_OPTIONS)
config.PREFIX = b'TESTXXXX'
util.FIRST_MULTISIG_BLOCK_TESTNET = 1
checkpoints = dict(check.CHECKPOINTS_TESTNET)
check.CHECKPOINTS_TESTNET = {}
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger_buff = io.StringIO()
handler = logging.StreamHandler(logger_buff)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
asyncio_log = logging.getLogger('asyncio')
asyncio_log.setLevel(logging.ERROR)
db = database.get_connection(read_only=False)
initialise_db(db)
raw_transactions = []
for tx in scenario:
if tx[0] != 'create_next_block':
module = sys.modules['lib.messages.{}'.format(tx[0])]
compose = getattr(module, 'compose')
unsigned_tx_hex = transaction.construct(db, get_proxy(),
compose(db, *tx[1]),
**tx[2])
raw_transactions.append({tx[0]: unsigned_tx_hex})
insert_raw_transaction(unsigned_tx_hex, db, rawtransactions_db)
else:
create_next_block(db,
block_index=config.BURN_START + tx[1],
parse_block=True)
dump = dump_database(db)
log = logger_buff.getvalue()
db.close()
check.CHECKPOINTS_TESTNET = checkpoints
return dump, log, json.dumps(raw_transactions, indent=4)
def setup_function(function):
counterpartyd.set_options(rpc_port=9999,
data_dir=tempfile.gettempdir(),
database_file=tempfile.gettempdir() +
'/counterpartyd.unittest.db',
rpc_password='******',
backend_rpc_password='******',
testnet=True,
testcoin=False,
backend_rpc_ssl_verify=False)
try:
os.remove(config.DATABASE)
except:
pass
# Connect to database.
global db
db = database.get_connection(read_only=False, foreign_keys=False)
from lib import blocks
blocks.initialise(db)
def run_scenario(scenario, rawtransactions_db):
counterpartyd.set_options(database_file=':memory:', testnet=True, **COUNTERPARTYD_OPTIONS)
config.PREFIX = b'TESTXXXX'
util.FIRST_MULTISIG_BLOCK_TESTNET = 1
checkpoints = dict(check.CHECKPOINTS_TESTNET)
check.CHECKPOINTS_TESTNET = {}
proxy = RpcProxy()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger_buff = io.StringIO()
handler = logging.StreamHandler(logger_buff)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
asyncio_log = logging.getLogger('asyncio')
asyncio_log.setLevel(logging.ERROR)
db = database.get_connection(read_only=False)
initialise_db(db)
raw_transactions = []
for transaction in scenario:
if transaction[0] != 'create_next_block':
module = sys.modules['lib.messages.{}'.format(transaction[0])]
compose = getattr(module, 'compose')
unsigned_tx_hex = bitcoin.transaction(db, compose(db, *transaction[1]), **transaction[2])
raw_transactions.append({transaction[0]: unsigned_tx_hex})
insert_raw_transaction(unsigned_tx_hex, db, rawtransactions_db)
else:
create_next_block(db, block_index=config.BURN_START + transaction[1], parse_block=True)
dump = dump_database(db)
log = logger_buff.getvalue()
db.close()
check.CHECKPOINTS_TESTNET = checkpoints
return dump, log, json.dumps(raw_transactions, indent=4)
def run(self):
logger.debug('Starting API Status Poller.')
global current_api_status_code, current_api_status_response_json
db = database.get_connection(integrity_check=False)
while self.stop_event.is_set() != True:
try:
# Check version.
if time.time(
) - self.last_version_check >= 60 * 60: # One hour since last check.
logger.debug('Checking version.')
code = 10
check.version(util.last_block(db)['block_index'])
self.last_version_check = time.time()
# Check that bitcoind is running, communicable, and caught up with the blockchain.
# Check that the database has caught up with bitcoind.
if time.time(
) - self.last_database_check > 10 * 60: # Ten minutes since last check.
code = 11
logger.debug('Checking backend state.')
check.backend_state(self.proxy)
code = 12
logger.debug('Checking database state.')
check.database_state(db, backend.getblockcount(self.proxy))
self.last_database_check = time.time()
except (check.VersionError, check.BackendError,
exceptions.DatabaseError) as e:
exception_name = e.__class__.__name__
exception_text = str(e)
logger.debug("API Status Poller: %s", exception_text)
jsonrpc_response = jsonrpc.exceptions.JSONRPCServerError(
message=exception_name, data=exception_text)
current_api_status_code = code
current_api_status_response_json = jsonrpc_response.json.encode(
)
else:
current_api_status_code = None
current_api_status_response_json = None
time.sleep(config.BACKEND_POLL_INTERVAL)
def counterpartyd_db(request):
db = database.get_connection(read_only=False)
cursor = db.cursor()
cursor.execute('''BEGIN''')
request.addfinalizer(lambda: cursor.execute('''ROLLBACK'''))
return db
import settings
import logging
import datetime
from lib import database, weatherdata
from WeatherService import app
# setup the logging
logFormatStr = '[%(asctime)s] p%(process)s {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s'
logging.basicConfig(format = logFormatStr, filename = settings.LOG_FILE, level=settings.LOG_LEVEL)
formatter = logging.Formatter(logFormatStr,'%m-%d %H:%M:%S')
streamHandler = logging.StreamHandler()
streamHandler.setLevel(logging.DEBUG)
streamHandler.setFormatter(formatter)
app.logger.addHandler(streamHandler)
# create database if not exists - just an open connection needed for that
conn = database.get_connection()
try:
conn = database.get_connection()
except:
app.logger.error('Can\'t connect to the database.')
# request data from weather webservice and stores in the db
for city in settings.OPENWEATHERMAP_CITIES:
resp = weatherdata.query_weather_data(city)
weatherdata.load_data(resp)
def reparse(testnet=True):
options = dict(COUNTERPARTYD_OPTIONS)
options.pop('data_dir')
counterpartyd.set_options(database_file=':memory:', testnet=testnet, **options)
if testnet:
config.PREFIX = b'TESTXXXX'
logger = logging.getLogger()
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logger.addHandler(console)
memory_db = database.get_connection(read_only=False)
initialise_db(memory_db)
prod_db_path = os.path.join(config.DATA_DIR, '{}.{}{}.db'.format(config.XCP_CLIENT, str(config.VERSION_MAJOR), '.testnet' if testnet else ''))
prod_db = apsw.Connection(prod_db_path)
prod_db.setrowtrace(database.rowtracer)
with memory_db.backup("main", prod_db, "main") as backup:
backup.step()
# here we don't use block.reparse() because it reparse db in transaction (`with db`)
memory_cursor = memory_db.cursor()
for table in blocks.TABLES + ['balances']:
memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))
# clean consensus hashes if first block hash don't match with checkpoint.
checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
columns = [column['name'] for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')]
for field in ['ledger_hash', 'txlist_hash']:
if field in columns:
sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(field)
first_hash = list(memory_cursor.execute(sql, (config.BLOCK_FIRST,)))[0][field]
if first_hash != checkpoints[config.BLOCK_FIRST][field]:
logging.info('First hash changed. Cleaning {}.'.format(field))
memory_cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field))
blocks.initialise(memory_db)
previous_ledger_hash = None
previous_txlist_hash = None
memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
for block in memory_cursor.fetchall():
try:
logger.info('Block (re‐parse): {}'.format(str(block['block_index'])))
previous_ledger_hash, previous_txlist_hash = blocks.parse_block(memory_db, block['block_index'], block['block_time'],
previous_ledger_hash, block['ledger_hash'],
previous_txlist_hash, block['txlist_hash'])
except check.ConsensusError as e:
message = str(e)
if message.find('ledger_hash') != -1:
new_ledger = get_block_ledger(memory_db, block['block_index'])
old_ledger = get_block_ledger(prod_db, block['block_index'])
compare_strings(old_ledger, new_ledger)
elif message.find('txlist_hash') != -1:
new_txlist = get_block_txlist(memory_db, block['block_index'])
old_txlist = get_block_txlist(prod_db, block['block_index'])
compare_strings(old_txlist, new_txlist)
raise(e)
def run(self):
logger.debug('Starting API Server.')
db = database.get_connection(integrity_check=False)
app = flask.Flask(__name__)
auth = HTTPBasicAuth()
@auth.get_password
def get_pw(username):
if username == config.RPC_USER:
return config.RPC_PASSWORD
return None
######################
#READ API
# Generate dynamically get_{table} methods
def generate_get_method(table):
def get_method(**kwargs):
try:
return get_rows(db, table=table, **kwargs)
except TypeError as e: #TODO: generalise for all API methods
raise APIError(str(e))
return get_method
for table in API_TABLES:
new_method = generate_get_method(table)
new_method.__name__ = 'get_{}'.format(table)
dispatcher.add_method(new_method)
@dispatcher.add_method
def sql(query, bindings=None):
if bindings == None:
bindings = []
return db_query(db, query, tuple(bindings))
######################
#WRITE/ACTION API
# Generate dynamically create_{transaction} and do_{transaction} methods
def generate_create_method(tx):
def split_params(**kwargs):
transaction_args = {}
common_args = {}
private_key_wif = None
for key in kwargs:
if key in COMMONS_ARGS:
common_args[key] = kwargs[key]
elif key == 'privkey':
private_key_wif = kwargs[key]
else:
transaction_args[key] = kwargs[key]
return transaction_args, common_args, private_key_wif
def create_method(**kwargs):
try:
transaction_args, common_args, private_key_wif = split_params(
**kwargs)
return compose_transaction(db,
self.proxy,
name=tx,
params=transaction_args,
**common_args)
except TypeError as e: #TODO: generalise for all API methods
raise APIError(str(e))
def do_method(**kwargs):
try:
transaction_args, common_args, private_key_wif = split_params(
**kwargs)
return do_transaction(db,
self.proxy,
name=tx,
params=transaction_args,
private_key_wif=private_key_wif,
**common_args)
except TypeError as e: #TODO: generalise for all API methods
raise APIError(str(e))
return create_method, do_method
for tx in API_TRANSACTIONS:
create_method, do_method = generate_create_method(tx)
create_method.__name__ = 'create_{}'.format(tx)
do_method.__name__ = 'do_{}'.format(tx)
dispatcher.add_method(create_method)
dispatcher.add_method(do_method)
@dispatcher.add_method
def sign_tx(unsigned_tx_hex, privkey=None):
return sign_transaction(self.proxy,
unsigned_tx_hex,
private_key_wif=privkey)
@dispatcher.add_method
def broadcast_tx(signed_tx_hex):
return broadcast_transaction(self.proxy, signed_tx_hex)
@dispatcher.add_method
def get_messages(block_index):
if not isinstance(block_index, int):
raise APIError("block_index must be an integer.")
cursor = db.cursor()
cursor.execute(
'select * from messages where block_index = ? order by message_index asc',
(block_index, ))
messages = cursor.fetchall()
cursor.close()
return messages
@dispatcher.add_method
def get_messages_by_index(message_indexes):
"""Get specific messages from the feed, based on the message_index.
@param message_index: A single index, or a list of one or more message indexes to retrieve.
"""
if not isinstance(message_indexes, list):
message_indexes = [
message_indexes,
]
for idx in message_indexes: #make sure the data is clean
if not isinstance(idx, int):
raise APIError(
"All items in message_indexes are not integers")
cursor = db.cursor()
cursor.execute(
'SELECT * FROM messages WHERE message_index IN (%s) ORDER BY message_index ASC'
% (','.join([str(x) for x in message_indexes]), ))
messages = cursor.fetchall()
cursor.close()
return messages
@dispatcher.add_method
def get_xcp_supply():
return util.xcp_supply(db)
@dispatcher.add_method
def get_asset_info(assets):
if not isinstance(assets, list):
raise APIError(
"assets must be a list of asset names, even if it just contains one entry"
)
assetsInfo = []
for asset in assets:
# BTC and XCP.
if asset in [config.BTC, config.XCP]:
if asset == config.BTC:
supply = backend.get_btc_supply(self.proxy,
normalize=False)
else:
supply = util.xcp_supply(db)
assetsInfo.append({
'asset': asset,
'owner': None,
'divisible': True,
'locked': False,
'supply': supply,
'description': '',
'issuer': None
})
continue
# User‐created asset.
cursor = db.cursor()
issuances = list(
cursor.execute(
'''SELECT * FROM issuances WHERE (status = ? AND asset = ?) ORDER BY block_index ASC''',
('valid', asset)))
cursor.close()
if not issuances:
continue #asset not found, most likely
else:
last_issuance = issuances[-1]
locked = False
for e in issuances:
if e['locked']: locked = True
assetsInfo.append({
'asset': asset,
'owner': last_issuance['issuer'],
'divisible': bool(last_issuance['divisible']),
'locked': locked,
'supply': util.asset_supply(db, asset),
'description': last_issuance['description'],
'issuer': last_issuance['issuer']
})
return assetsInfo
@dispatcher.add_method
def get_block_info(block_index):
assert isinstance(block_index, int)
cursor = db.cursor()
cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''',
(block_index, ))
blocks = list(cursor)
if len(blocks) == 1:
block = blocks[0]
elif len(blocks) == 0:
raise exceptions.DatabaseError('No blocks found.')
else:
assert False
cursor.close()
return block
@dispatcher.add_method
def get_blocks(block_indexes):
"""fetches block info and messages for the specified block indexes"""
if not isinstance(block_indexes, (list, tuple)):
raise APIError("block_indexes must be a list of integers.")
if len(block_indexes) >= 250:
raise APIError("can only specify up to 250 indexes at a time.")
block_indexes_str = ','.join([str(x) for x in block_indexes])
cursor = db.cursor()
cursor.execute(
'SELECT * FROM blocks WHERE block_index IN (%s) ORDER BY block_index ASC'
% (block_indexes_str, ))
blocks = cursor.fetchall()
cursor.execute(
'SELECT * FROM messages WHERE block_index IN (%s) ORDER BY block_index ASC, message_index ASC'
% (block_indexes_str, ))
messages = collections.deque(cursor.fetchall())
for block in blocks:
# messages_in_block = []
block['_messages'] = []
while len(messages) and messages[0]['block_index'] == block[
'block_index']:
block['_messages'].append(messages.popleft())
assert not len(messages) #should have been cleared out
cursor.close()
return blocks
@dispatcher.add_method
def get_running_info():
latestBlockIndex = backend.getblockcount(self.proxy)
try:
check.database_state(db, latestBlockIndex)
except exceptions.DatabaseError:
caught_up = False
else:
caught_up = True
try:
last_block = util.last_block(db)
except:
last_block = {
'block_index': None,
'block_hash': None,
'block_time': None
}
try:
last_message = util.last_message(db)
except:
last_message = None
return {
'db_caught_up':
caught_up,
'bitcoin_block_count':
latestBlockIndex,
'last_block':
last_block,
'last_message_index':
last_message['message_index'] if last_message else -1,
'running_testnet':
config.TESTNET,
'running_testcoin':
config.TESTCOIN,
'version_major':
config.VERSION_MAJOR,
'version_minor':
config.VERSION_MINOR,
'version_revision':
config.VERSION_REVISION
}
@dispatcher.add_method
def get_element_counts():
counts = {}
cursor = db.cursor()
for element in [
'transactions', 'blocks', 'debits', 'credits', 'balances',
'sends', 'orders', 'order_matches', 'btcpays', 'issuances',
'broadcasts', 'bets', 'bet_matches', 'dividends', 'burns',
'cancels', 'order_expirations', 'bet_expirations',
'order_match_expirations', 'bet_match_expirations',
'messages'
]:
cursor.execute("SELECT COUNT(*) AS count FROM %s" % element)
count_list = cursor.fetchall()
assert len(count_list) == 1
counts[element] = count_list[0]['count']
cursor.close()
return counts
@dispatcher.add_method
def get_asset_names():
cursor = db.cursor()
names = [
row['asset'] for row in cursor.execute(
"SELECT DISTINCT asset FROM issuances WHERE status = 'valid' ORDER BY asset ASC"
)
]
cursor.close()
return names
@dispatcher.add_method
def get_holder_count(asset):
holders = util.holders(db, asset)
addresses = []
for holder in holders:
addresses.append(holder['address'])
return {asset: len(set(addresses))}
@dispatcher.add_method
def search_raw_transactions(address):
return blockchain.searchrawtransactions(self.proxy, address)
@dispatcher.add_method
def get_unspent_txouts(address, return_confirmed=False):
result = backend.get_unspent_txouts(
self.proxy, address, return_confirmed=return_confirmed)
if return_confirmed:
return {'all': result[0], 'confirmed': result[1]}
else:
return result
@dispatcher.add_method
def get_wallet():
# TODO: Dupe with `backend.get_wallet()`
wallet = {}
for group in backend.listaddressgroupings(self.proxy):
for bunch in group:
address, btc_balance = bunch[:2]
wallet[address] = str(btc_balance)
return wallet
@dispatcher.add_method
def get_tx_info(tx_hex):
source, destination, btc_amount, fee, data = blocks.get_tx_info(
self.proxy, tx_hex,
util.last_block(db)['block_index'])
return source, destination, btc_amount, fee, util.hexlify(data)
@dispatcher.add_method
def unpack(data_hex):
data = binascii.unhexlify(data_hex)
message_type_id = struct.unpack(config.TXTYPE_FORMAT, data[:4])[0]
message = data[4:]
for message_type in API_TRANSACTIONS:
if message_type_id == sys.modules['lib.messages.{}'.format(
message_type)].ID:
unpack_method = sys.modules['lib.messages.{}'.format(
message_type)].unpack
unpacked = unpack_method(
db, message,
util.last_block(db)['block_index'])
return message_type_id, unpacked
def _set_cors_headers(response):
if config.RPC_ALLOW_CORS:
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers[
'Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
response.headers[
'Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'
@app.route('/', methods=[
"OPTIONS",
])
@app.route('/api/', methods=[
"OPTIONS",
])
def handle_options():
response = flask.Response('', 204)
_set_cors_headers(response)
return response
@app.route('/', methods=[
"POST",
])
@app.route('/api/', methods=[
"POST",
])
@auth.login_required
def handle_post():
try:
request_json = flask.request.get_data().decode('utf-8')
request_data = json.loads(request_json)
assert 'id' in request_data and request_data[
'jsonrpc'] == "2.0" and request_data['method']
# params may be omitted
except:
obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest(
data="Invalid JSON-RPC 2.0 request format")
return flask.Response(obj_error.json.encode(),
200,
mimetype='application/json')
#only arguments passed as a dict are supported
if request_data.get('params', None) and not isinstance(
request_data['params'], dict):
obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest(
data=
'Arguments must be passed as a JSON object (list of unnamed arguments not supported)'
)
return flask.Response(obj_error.json.encode(),
200,
mimetype='application/json')
#return an error if API fails checks
if not config.FORCE and current_api_status_code:
return flask.Response(current_api_status_response_json,
200,
mimetype='application/json')
jsonrpc_response = jsonrpc.JSONRPCResponseManager.handle(
request_json, dispatcher)
response = flask.Response(jsonrpc_response.json.encode(),
200,
mimetype='application/json')
_set_cors_headers(response)
return response
init_api_access_log()
http_server = HTTPServer(WSGIContainer(app), xheaders=True)
try:
http_server.listen(config.RPC_PORT, address=config.RPC_HOST)
self.is_ready = True
self.ioloop.start()
except OSError:
raise APIError(
"Cannot start the API subsystem. Is {} already running, or is something else listening on port {}?"
.format(config.XCP_CLIENT, config.RPC_PORT))
db.close()
http_server.stop()
self.ioloop.close()
return
def reparse(testnet=True):
options = dict(COUNTERPARTYD_OPTIONS)
options.pop('data_dir')
counterpartyd.set_options(database_file=':memory:',
testnet=testnet,
**options)
if testnet:
config.PREFIX = b'TESTXXXX'
logger = logging.getLogger()
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logger.addHandler(console)
memory_db = database.get_connection(read_only=False)
initialise_db(memory_db)
prod_db_path = os.path.join(
config.DATA_DIR, '{}.{}{}.db'.format(config.XCP_CLIENT,
str(config.VERSION_MAJOR),
'.testnet' if testnet else ''))
prod_db = apsw.Connection(prod_db_path)
prod_db.setrowtrace(database.rowtracer)
with memory_db.backup("main", prod_db, "main") as backup:
backup.step()
# here we don't use block.reparse() because it reparse db in transaction (`with db`)
memory_cursor = memory_db.cursor()
for table in blocks.TABLES + ['balances']:
memory_cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))
# clean consensus hashes if first block hash don't match with checkpoint.
checkpoints = check.CHECKPOINTS_TESTNET if config.TESTNET else check.CHECKPOINTS_MAINNET
columns = [
column['name']
for column in memory_cursor.execute('''PRAGMA table_info(blocks)''')
]
for field in ['ledger_hash', 'txlist_hash']:
if field in columns:
sql = '''SELECT {} FROM blocks WHERE block_index = ?'''.format(
field)
first_hash = list(
memory_cursor.execute(sql, (config.BLOCK_FIRST, )))[0][field]
if first_hash != checkpoints[config.BLOCK_FIRST][field]:
logger.info('First hash changed. Cleaning {}.'.format(field))
memory_cursor.execute(
'''UPDATE blocks SET {} = NULL'''.format(field))
blocks.initialise(memory_db)
previous_ledger_hash = None
previous_txlist_hash = None
memory_cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
for block in memory_cursor.fetchall():
try:
logger.info('Block (re‐parse): {}'.format(str(
block['block_index'])))
previous_ledger_hash, previous_txlist_hash = blocks.parse_block(
memory_db, block['block_index'], block['block_time'],
previous_ledger_hash, block['ledger_hash'],
previous_txlist_hash, block['txlist_hash'])
except check.ConsensusError as e:
message = str(e)
if message.find('ledger_hash') != -1:
new_ledger = get_block_ledger(memory_db, block['block_index'])
old_ledger = get_block_ledger(prod_db, block['block_index'])
compare_strings(old_ledger, new_ledger)
elif message.find('txlist_hash') != -1:
new_txlist = get_block_txlist(memory_db, block['block_index'])
old_txlist = get_block_txlist(prod_db, block['block_index'])
compare_strings(old_txlist, new_txlist)
raise (e)
# Version
if args.action in ('server', 'reparse', 'rollback') and not config.FORCE:
logger.info('Checking version.')
try:
check.version(backend.getblockcount(proxy))
except check.VersionUpdateRequiredError as e:
traceback.print_exc(file=sys.stdout)
sys.exit(config.EXITCODE_UPDATE_REQUIRED)
# Lock
if args.action in ('rollback', 'reparse', 'server', 'kickstart') and not config.FORCE:
get_lock()
# Database
logger.info('Connecting to database.')
db = database.get_connection(read_only=False)
# MESSAGE CREATION
if args.action == 'send':
if args.fee:
args.fee = util.value_in(db, args.fee, config.BTC)
quantity = util.value_in(db, args.quantity, args.asset)
cli('create_send', {'source': args.source,
'destination': args.destination, 'asset':
args.asset, 'quantity': quantity, 'fee': args.fee,
'allow_unconfirmed_inputs': args.unconfirmed,
'encoding': args.encoding, 'fee_per_kb':
args.fee_per_kb, 'regular_dust_size':
args.regular_dust_size, 'multisig_dust_size':
args.multisig_dust_size, 'op_return_value':
args.op_return_value},
parser.add_argument(
'-v', '--verbose',
action='count',
default=0,
help='increase verbosity')
parser.add_argument(
'-n', '--no-history',
action='store_true',
help='do not collect historical data')
args = parser.parse_args()
if args.verbose > 1:
loglevel = logging.DEBUG
elif args.verbose > 0:
loglevel = logging.INFO
else:
loglevel = logging.WARNING
logging.basicConfig(level=loglevel, format=LOGFORMAT)
config = Config(args.config)
cloudtrax = CloudTrax(config)
cloudtrax.collect_networks()
cloudtrax.collect_nodes()
if not args.no_history:
cloudtrax.collect_node_history()
cloudtrax.collect_clients()
dbconf = config.get_db()
database = get_connection(dbconf['type'], **dbconf)
database.store_data(cloudtrax)
def run(self):
logger.debug('Starting API Server.')
db = database.get_connection(integrity_check=False)
app = flask.Flask(__name__)
auth = HTTPBasicAuth()
@auth.get_password
def get_pw(username):
if username == config.RPC_USER:
return config.RPC_PASSWORD
return None
######################
#READ API
# Generate dynamically get_{table} methods
def generate_get_method(table):
def get_method(**kwargs):
try:
return get_rows(db, table=table, **kwargs)
except TypeError as e: #TODO: generalise for all API methods
raise APIError(str(e))
return get_method
for table in API_TABLES:
new_method = generate_get_method(table)
new_method.__name__ = 'get_{}'.format(table)
dispatcher.add_method(new_method)
@dispatcher.add_method
def sql(query, bindings=None):
if bindings == None:
bindings = []
return db_query(db, query, tuple(bindings))
######################
#WRITE/ACTION API
# Generate dynamically create_{transaction} and do_{transaction} methods
def generate_create_method(tx):
def split_params(**kwargs):
transaction_args = {}
common_args = {}
private_key_wif = None
for key in kwargs:
if key in COMMONS_ARGS:
common_args[key] = kwargs[key]
elif key == 'privkey':
private_key_wif = kwargs[key]
else:
transaction_args[key] = kwargs[key]
return transaction_args, common_args, private_key_wif
def create_method(**kwargs):
try:
transaction_args, common_args, private_key_wif = split_params(**kwargs)
return compose_transaction(db, self.proxy, name=tx, params=transaction_args, **common_args)
except TypeError as e: #TODO: generalise for all API methods
raise APIError(str(e))
def do_method(**kwargs):
try:
transaction_args, common_args, private_key_wif = split_params(**kwargs)
return do_transaction(db, self.proxy, name=tx, params=transaction_args, private_key_wif=private_key_wif, **common_args)
except TypeError as e: #TODO: generalise for all API methods
raise APIError(str(e))
return create_method, do_method
for tx in API_TRANSACTIONS:
create_method, do_method = generate_create_method(tx)
create_method.__name__ = 'create_{}'.format(tx)
do_method.__name__ = 'do_{}'.format(tx)
dispatcher.add_method(create_method)
dispatcher.add_method(do_method)
@dispatcher.add_method
def sign_tx(unsigned_tx_hex, privkey=None):
return sign_transaction(self.proxy, unsigned_tx_hex, private_key_wif=privkey)
@dispatcher.add_method
def broadcast_tx(signed_tx_hex):
return broadcast_transaction(self.proxy, signed_tx_hex)
@dispatcher.add_method
def get_messages(block_index):
if not isinstance(block_index, int):
raise APIError("block_index must be an integer.")
cursor = db.cursor()
cursor.execute('select * from messages where block_index = ? order by message_index asc', (block_index,))
messages = cursor.fetchall()
cursor.close()
return messages
@dispatcher.add_method
def get_messages_by_index(message_indexes):
"""Get specific messages from the feed, based on the message_index.
@param message_index: A single index, or a list of one or more message indexes to retrieve.
"""
if not isinstance(message_indexes, list):
message_indexes = [message_indexes,]
for idx in message_indexes: #make sure the data is clean
if not isinstance(idx, int):
raise APIError("All items in message_indexes are not integers")
cursor = db.cursor()
cursor.execute('SELECT * FROM messages WHERE message_index IN (%s) ORDER BY message_index ASC'
% (','.join([str(x) for x in message_indexes]),))
messages = cursor.fetchall()
cursor.close()
return messages
@dispatcher.add_method
def get_xcp_supply():
return util.xcp_supply(db)
@dispatcher.add_method
def get_asset_info(assets):
if not isinstance(assets, list):
raise APIError("assets must be a list of asset names, even if it just contains one entry")
assetsInfo = []
for asset in assets:
# BTC and XCP.
if asset in [config.BTC, config.XCP]:
if asset == config.BTC:
supply = backend.get_btc_supply(self.proxy, normalize=False)
else:
supply = util.xcp_supply(db)
assetsInfo.append({
'asset': asset,
'owner': None,
'divisible': True,
'locked': False,
'supply': supply,
'description': '',
'issuer': None
})
continue
# User‐created asset.
cursor = db.cursor()
issuances = list(cursor.execute('''SELECT * FROM issuances WHERE (status = ? AND asset = ?) ORDER BY block_index ASC''', ('valid', asset)))
cursor.close()
if not issuances:
continue #asset not found, most likely
else:
last_issuance = issuances[-1]
locked = False
for e in issuances:
if e['locked']: locked = True
assetsInfo.append({
'asset': asset,
'owner': last_issuance['issuer'],
'divisible': bool(last_issuance['divisible']),
'locked': locked,
'supply': util.asset_supply(db, asset),
'description': last_issuance['description'],
'issuer': last_issuance['issuer']})
return assetsInfo
@dispatcher.add_method
def get_block_info(block_index):
assert isinstance(block_index, int)
cursor = db.cursor()
cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,))
blocks = list(cursor)
if len(blocks) == 1:
block = blocks[0]
elif len(blocks) == 0:
raise exceptions.DatabaseError('No blocks found.')
else:
assert False
cursor.close()
return block
@dispatcher.add_method
def get_blocks(block_indexes):
"""fetches block info and messages for the specified block indexes"""
if not isinstance(block_indexes, (list, tuple)):
raise APIError("block_indexes must be a list of integers.")
if len(block_indexes) >= 250:
raise APIError("can only specify up to 250 indexes at a time.")
block_indexes_str = ','.join([str(x) for x in block_indexes])
cursor = db.cursor()
cursor.execute('SELECT * FROM blocks WHERE block_index IN (%s) ORDER BY block_index ASC'
% (block_indexes_str,))
blocks = cursor.fetchall()
cursor.execute('SELECT * FROM messages WHERE block_index IN (%s) ORDER BY block_index ASC, message_index ASC'
% (block_indexes_str,))
messages = collections.deque(cursor.fetchall())
for block in blocks:
# messages_in_block = []
block['_messages'] = []
while len(messages) and messages[0]['block_index'] == block['block_index']:
block['_messages'].append(messages.popleft())
assert not len(messages) #should have been cleared out
cursor.close()
return blocks
@dispatcher.add_method
def get_running_info():
latestBlockIndex = backend.getblockcount(self.proxy)
try:
check.database_state(db, latestBlockIndex)
except exceptions.DatabaseError:
caught_up = False
else:
caught_up = True
try:
last_block = util.last_block(db)
except:
last_block = {'block_index': None, 'block_hash': None, 'block_time': None}
try:
last_message = util.last_message(db)
except:
last_message = None
return {
'db_caught_up': caught_up,
'bitcoin_block_count': latestBlockIndex,
'last_block': last_block,
'last_message_index': last_message['message_index'] if last_message else -1,
'running_testnet': config.TESTNET,
'running_testcoin': config.TESTCOIN,
'version_major': config.VERSION_MAJOR,
'version_minor': config.VERSION_MINOR,
'version_revision': config.VERSION_REVISION
}
@dispatcher.add_method
def get_element_counts():
counts = {}
cursor = db.cursor()
for element in ['transactions', 'blocks', 'debits', 'credits', 'balances', 'sends', 'orders',
'order_matches', 'btcpays', 'issuances', 'broadcasts', 'bets', 'bet_matches', 'dividends',
'burns', 'cancels', 'order_expirations', 'bet_expirations', 'order_match_expirations',
'bet_match_expirations', 'messages']:
cursor.execute("SELECT COUNT(*) AS count FROM %s" % element)
count_list = cursor.fetchall()
assert len(count_list) == 1
counts[element] = count_list[0]['count']
cursor.close()
return counts
@dispatcher.add_method
def get_asset_names():
cursor = db.cursor()
names = [row['asset'] for row in cursor.execute("SELECT DISTINCT asset FROM issuances WHERE status = 'valid' ORDER BY asset ASC")]
cursor.close()
return names
@dispatcher.add_method
def get_holder_count(asset):
holders = util.holders(db, asset)
addresses = []
for holder in holders:
addresses.append(holder['address'])
return {asset: len(set(addresses))}
@dispatcher.add_method
def search_raw_transactions(address):
return blockchain.searchrawtransactions(self.proxy, address)
@dispatcher.add_method
def get_unspent_txouts(address, return_confirmed=False):
result = backend.get_unspent_txouts(self.proxy, address, return_confirmed=return_confirmed)
if return_confirmed:
return {'all': result[0], 'confirmed': result[1]}
else:
return result
@dispatcher.add_method
def get_wallet():
# TODO: Dupe with `backend.get_wallet()`
wallet = {}
for group in backend.listaddressgroupings(self.proxy):
for bunch in group:
address, btc_balance = bunch[:2]
wallet[address] = str(btc_balance)
return wallet
@dispatcher.add_method
def get_tx_info(tx_hex):
source, destination, btc_amount, fee, data = blocks.get_tx_info(tx_hex, util.last_block(db)['block_index'])
return source, destination, btc_amount, fee, util.hexlify(data)
@dispatcher.add_method
def unpack(data_hex):
data = binascii.unhexlify(data_hex)
message_type_id = struct.unpack(config.TXTYPE_FORMAT, data[:4])[0]
message = data[4:]
for message_type in API_TRANSACTIONS:
if message_type_id == sys.modules['lib.messages.{}'.format(message_type)].ID:
unpack_method = sys.modules['lib.messages.{}'.format(message_type)].unpack
unpacked = unpack_method(db, message, util.last_block(db)['block_index'])
return message_type_id, unpacked
def _set_cors_headers(response):
if config.RPC_ALLOW_CORS:
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
response.headers['Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'
@app.route('/', methods=["OPTIONS",])
@app.route('/api/', methods=["OPTIONS",])
def handle_options():
response = flask.Response('', 204)
_set_cors_headers(response)
return response
@app.route('/', methods=["POST",])
@app.route('/api/', methods=["POST",])
@auth.login_required
def handle_post():
try:
request_json = flask.request.get_data().decode('utf-8')
request_data = json.loads(request_json)
assert 'id' in request_data and request_data['jsonrpc'] == "2.0" and request_data['method']
# params may be omitted
except:
obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest(data="Invalid JSON-RPC 2.0 request format")
return flask.Response(obj_error.json.encode(), 200, mimetype='application/json')
#only arguments passed as a dict are supported
if request_data.get('params', None) and not isinstance(request_data['params'], dict):
obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest(
data='Arguments must be passed as a JSON object (list of unnamed arguments not supported)')
return flask.Response(obj_error.json.encode(), 200, mimetype='application/json')
#return an error if API fails checks
if not config.FORCE and current_api_status_code:
return flask.Response(current_api_status_response_json, 200, mimetype='application/json')
jsonrpc_response = jsonrpc.JSONRPCResponseManager.handle(request_json, dispatcher)
response = flask.Response(jsonrpc_response.json.encode(), 200, mimetype='application/json')
_set_cors_headers(response)
return response
init_api_access_log()
http_server = HTTPServer(WSGIContainer(app), xheaders=True)
try:
http_server.listen(config.RPC_PORT, address=config.RPC_HOST)
self.is_ready = True
self.ioloop.start()
except OSError:
raise APIError("Cannot start the API subsystem. Is {} already running, or is something else listening on port {}?".format(config.XCP_CLIENT, config.RPC_PORT))
db.close()
http_server.stop()
self.ioloop.close()
return
