def core_loop(syncer, config):
message_queue = syncer.queues['Blockchain']
init_storage_space(config)
nodes = {}
set_ask_for_blocks_hook(storage_space.blockchain, message_queue)
set_ask_for_txouts_hook(storage_space.blocks_storage, message_queue)
requests = {}
message_queue.put({"action": "give nodes list reminder"})
def send_message(destination, message):
if not 'id' in message:
message['id'] = uuid4()
if not 'sender' in message:
message['sender'] = "Blockchain"
syncer.queues[destination].put(message)
def send_to_nm(message):
send_message("NetworkManager", message)
logger.debug("Start of core loop")
while True:
sleep(0.05)
put_back_messages = []
while not message_queue.empty():
message = message_queue.get()
if 'time' in message and message['time'] > time(
): # delay this message
put_back_messages.append(message)
continue
logger.info("Processing message %s" % message)
if not 'action' in message: #it is response
if message['id'] in requests: # response is awaited
if requests[message['id']] == "give nodes list":
requests.pop(message['id'])
message_queue.put({
"action": "take nodes list",
"nodes": message["result"]
})
else:
pass #Drop
continue
try:
if message["action"] == "take the headers":
process_new_headers(message)
if message["action"] == "take the blocks":
initial_tip = storage_space.blockchain.current_tip
process_new_blocks(message)
after_tip = storage_space.blockchain.current_tip
if not after_tip == initial_tip:
notify_all_nodes_about_new_tip(nodes, send_to_nm)
if message["action"] == "take the txos":
process_new_txos(message)
if message["action"] == "give blocks":
process_blocks_request(message, send_message)
if message["action"] == "give next headers":
process_next_headers_request(message, send_message)
if message["action"] == "give txos":
process_txos_request(message, send_message)
if message["action"] == "find common root":
process_find_common_root(message, send_message)
if message["action"] == "find common root response":
process_find_common_root_reponse(message,
nodes[message["node"]],
send_message)
if message["action"] == "give TBM transaction":
process_tbm_tx_request(message, send_message)
if message["action"] == "take TBM transaction":
process_tbm_tx(message, send_to_nm, nodes)
if message["action"] == "give tip height":
send_message(
message["sender"], {
"id": message["id"],
"result": storage_space.blockchain.current_height
})
if message["action"] == "take tip info":
if not message["node"] in nodes:
nodes[message["node"]] = {'node': message["node"]}
process_tip_info(message,
nodes[message["node"]],
send=send_to_nm)
except DOSException as e:
logger.info("DOS Exception %s" % str(e))
#raise e #TODO send to NM
except Exception as e:
raise e
if message["action"] == "give block template":
block = storage_space.mempool_tx.give_block_template()
ser_head = block.header.serialize()
send_message(message["sender"], {
"id": message["id"],
"result": ser_head
})
if message["action"] == "take solved block template":
try:
initial_tip = storage_space.blockchain.current_tip
header = Header()
header.deserialize(message["solved template"])
solved_block = storage_space.mempool_tx.get_block_by_header_solution(
header)
storage_space.headers_manager.add_header(
solved_block.header)
storage_space.headers_manager.context_validation(
solved_block.header.hash)
solved_block.non_context_verify()
storage_space.blockchain.add_block(solved_block)
send_message(message["sender"], {
"id": message["id"],
"result": "Accepted"
})
after_tip = storage_space.blockchain.current_tip
if not after_tip == initial_tip:
notify_all_nodes_about_new_tip(nodes, send_to_nm)
except Exception as e:
raise e
send_message(message["sender"], {
"id": message["id"],
"error": str(e)
})
if message["action"] == "get confirmed balance stats":
if storage_space.mempool_tx.key_manager:
stats = storage_space.mempool_tx.key_manager.get_confirmed_balance_stats(
storage_space.utxo_index, storage_space.txos_storage,
storage_space.blockchain.current_height)
send_message(message["sender"], {
"id": message["id"],
"result": stats
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "get confirmed balance list":
if storage_space.mempool_tx.key_manager:
_list = storage_space.mempool_tx.key_manager.get_confirmed_balance_list(
storage_space.utxo_index, storage_space.txos_storage,
storage_space.blockchain.current_height)
send_message(message["sender"], {
"id": message["id"],
"result": _list
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "give new address":
if storage_space.mempool_tx.key_manager:
texted_address = storage_space.mempool_tx.key_manager.new_address(
).to_text()
send_message(message["sender"], {
"id": message["id"],
"result": texted_address
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "give private key":
if storage_space.mempool_tx.key_manager:
km = storage_space.mempool_tx.key_manager
a = Address()
a.from_text(message["address"])
serialized_pk = km.priv_by_address(a).serialize()
send_message(message["sender"], {
"id": message["id"],
"result": serialized_pk
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "take private key":
if storage_space.mempool_tx.key_manager:
km = storage_space.mempool_tx.key_manager
pk = PrivateKey()
pk.deserialize(message['privkey'])
km.add_privkey(pk)
send_message(message["sender"], {
"id": message["id"],
"result": "imported"
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "give synchronization status":
our_height = storage_space.blockchain.current_height
best_known_header = storage_space.headers_manager.best_header_height
try:
best_advertised_height = max([
nodes[node]["height"] for node in nodes
if "height" in nodes[node]
])
except:
best_advertised_height = None
send_message(
message["sender"], {
"id": message["id"],
"result": {
'height': our_height,
'best_known_header': best_known_header,
'best_advertised_height': best_advertised_height
}
})
if message["action"] == "send to address":
value = int(message["value"])
taddress = message["address"]
a = Address()
a.from_text(taddress)
if storage_space.mempool_tx.key_manager:
_list = storage_space.mempool_tx.key_manager.get_confirmed_balance_list(
storage_space.utxo_index, storage_space.txos_storage,
storage_space.blockchain.current_height)
list_to_spend = []
summ = 0
for address in _list:
for texted_index in _list[address]:
if summ > value:
continue
if isinstance(_list[address][texted_index], int):
_index = base64.b64decode(
texted_index.encode())
utxo = storage_space.txos_storage.confirmed[
_index]
if not utxo.lock_height <= storage_space.blockchain.current_height:
continue
list_to_spend.append(utxo)
summ += _list[address][texted_index]
if summ < value:
send_message(
message["sender"], {
"id": message["id"],
"error": "Not enough matured coins"
})
tx = Transaction(
txos_storage=storage_space.txos_storage,
key_manager=storage_space.mempool_tx.key_manager)
for utxo in list_to_spend:
tx.push_input(utxo)
tx.add_destination((a, value))
tx.generate()
tx.verify()
storage_space.mempool_tx.add_tx(tx)
tx_skel = TransactionSkeleton(tx=tx)
notify_all_nodes_about_tx(tx_skel.serialize(
rich_format=True, max_size=40000),
nodes,
send_to_nm,
_except=[],
mode=1)
send_message(message["sender"], {
"id": message["id"],
"result": "generated"
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
#message from core_loop
if message["action"] == "check txouts download status":
txos = message["txos_hashes"]
to_be_downloaded = []
for txo in txos:
if not storage_space.txos_storage.known(txo):
to_be_downloaded.append(txo)
if not to_be_downloaded:
continue #We are good, txouts are already downloaded
already_asked_nodes = message["already_asked_nodes"]
asked = False
for node_params in nodes:
node = nodes[node_params]
if node in already_asked_nodes:
continue
already_asked_nodes += [node]
send_to_nm({
"action": "give txos",
"txos_hashes": b"".join(to_be_downloaded),
"num": len(to_be_downloaded),
"id": str(uuid4()),
"node": node_params
})
new_message = {
"action": "check txouts download status",
"txos_hashes": to_be_downloaded,
"already_asked_nodes": already_asked_nodes,
"id": str(uuid4()),
"time": int(time() + 300)
}
asked = True
put_back_messages.append(new_message)
break
if not asked: #We already asked all applicable nodes
message["time"] = int(time()) + 3600
message["already_asked_nodes"] = []
put_back_messages.append(
message) # we will try to ask again in an hour
#message from core_loop
if message["action"] == "check blocks download status":
#TODO download many blocks at once
block_hashes = message["block_hashes"]
to_be_downloaded = []
lowest_height = 1e10
for block_hash in block_hashes:
if block_hash in storage_space.blocks_storage:
continue #We are good, block already downloaded
if not block_hash in storage_space.blockchain.awaited_blocks:
continue #For some reason we don't need this block anymore
to_be_downloaded.append(block_hash)
if storage_space.headers_storage[
block_hash].height < lowest_height:
lowest_height = storage_space.headers_storage[
block_hash].height
already_asked_nodes = message["already_asked_nodes"]
asked = False
for node_params in nodes:
node = nodes[node_params]
if node in already_asked_nodes:
continue
if node["height"] < lowest_height:
continue
already_asked_nodes += [node]
send_to_nm({
"action": "give blocks",
"block_hashes": bytes(b"".join(block_hashes)),
'num': len(block_hashes),
"id": str(uuid4()),
"node": node_params
})
new_message = {
"action": "check blocks download status",
"block_hashes": to_be_downloaded,
"already_asked_nodes": already_asked_nodes,
"id": str(uuid4()),
"time": int(time() + 300)
}
asked = True
put_back_messages.append(new_message)
break
if not asked: #We already asked all applicable nodes
message["time"] = int(time()) + 3600
message["already_asked_nodes"] = []
put_back_messages.append(
message) # we will try to ask again in an hour
if message["action"] == "take nodes list":
for node in message["nodes"]:
if not node in nodes: #Do not overwrite
nodes[node] = {"node": node}
disconnected_nodes = []
for existing_node in nodes:
if not existing_node in message["nodes"]:
disconnected_nodes.append(existing_node)
for dn in disconnected_nodes:
nodes.pop(dn)
if message["action"] == "give nodes list reminder":
_id = str(uuid4())
send_to_nm({
"action": "give nodes list",
"sender": "Blockchain",
"id": _id
})
requests[_id] = "give nodes list"
put_back_messages.append({
"action": "give nodes list reminder",
"time": int(time()) + 3
})
for _message in put_back_messages:
message_queue.put(_message)
try:
check_sync_status(nodes, send_to_nm)
except Exception as e:
logger.error(e)
def wallet(syncer, config):
'''
Wallet is synchronous service which holds private keys and information about
owned outputs. It provides information for transactions and block templates
generation.
'''
def get_height(timeout=2.5):
_id = str(uuid4())
syncer.queues['Notifications'].put({
'action': 'get',
'id': _id,
'key': 'blockchain height',
'sender': "Wallet"
})
message_queue = syncer.queues['Wallet']
start_time = time()
result = None
while True:
put_back = [
] #We wait for specific message, all others will wait for being processed
while not message_queue.empty():
message = message_queue.get()
if (not 'id' in message) or (not message['id'] == _id):
put_back.append(message)
continue
result = message['result']
break
for message in put_back:
message_queue.put(message)
if result:
break
sleep(0.01)
if time() - start_time > timeout:
raise KeyError
if result == 'error':
raise KeyError
return result['value']
notification_cache = {}
def notify(key, value, timestamp=None):
if (key in notification_cache) and (
notification_cache[key]['value'] == value
) and (time() - notification_cache[key]['timestamp']) < 5:
return #Do not spam notifications with the same values
message = {}
message['id'] = uuid4()
message['sender'] = "Wallet"
if not timestamp:
timestamp = time()
message['time'] = timestamp
message['action'] = "set"
message['key'] = key
message['value'] = value
syncer.queues["Notifications"].put(message)
notification_cache[key] = {'value': value, 'timestamp': timestamp}
#set logging
default_log_level = logging.INFO
if "logging" in config: #debug, info, warning, error, critical
loglevels = {
"debug": logging.DEBUG,
"info": logging.INFO,
"warning": logging.WARNING,
"error": logging.ERROR,
"critical": logging.CRITICAL
}
if "base" in config["logging"] and config["logging"][
"base"] in loglevels:
logger.setLevel(loglevels[config["logging"]["base"]])
if "wallet" in config["logging"] and config["logging"][
"wallet"] in loglevels:
#its ok to rewrite
logger.setLevel(loglevels[config["logging"]["wallet"]])
message_queue = syncer.queues['Wallet']
_path = config['location']['wallet']
try:
password = config['wallet'].get('password', None)
except:
password = None
km = KeyDB(path=_path, password=password)
with km.open() as conn:
cursor = conn.cursor()
apply_migrations(cursor)
notify('last wallet update', time())
while True:
sleep(0.01)
while not message_queue.empty():
message = message_queue.get()
if 'action' in message:
logger.info("Process message `%s`" % message['action'])
logger.debug("Process message %s" % message)
else:
logger.info("Process message %s" % message)
if not 'action' in message:
continue
if message['action'] == "process new block":
tx = Transaction(txos_storage=None, excesses_storage=None)
tx.deserialize(
message['tx'], rtx=None, skip_verification=True
) #skip_verification allows us to not provide rtx
block_height = message['height']
last_time_updated = None
with km.open() as conn:
cursor = conn.cursor()
for index in tx.inputs:
#Note it is not check whether output is unspent or not, we check that output is marked as our and unspent in our wallet
if km.is_unspent(index, cursor):
km.spend_output(index, block_height, cursor)
last_time_updated = time()
for _o in tx.outputs:
if km.is_owned_pubkey(_o.address.pubkey.serialize(),
cursor):
km.add_output(_o, block_height, cursor)
last_time_updated = time()
if km.is_saved(_o, cursor):
km.register_processed_output(
_o.serialized_index, block_height, cursor)
if last_time_updated:
notify('last wallet update', last_time_updated)
if message['action'] == "process rollback":
rollback = message['rollback_object']
block_height = message['block_height']
with km.open() as conn:
cursor = conn.cursor()
km.rollback(block_height, cursor)
last_time_updated = time()
notify('last wallet update', last_time_updated)
if message[
'action'] == "process indexed outputs": #during private key import correspondent outputs will be processed again
pass
if message['action'] == "give new taddress":
with km.open() as conn:
cursor = conn.cursor()
address = km.new_address(cursor)
response = {"id": message["id"], "result": address.to_text()}
syncer.queues[message['sender']].put(response)
if message['action'] == "give new address":
with km.open() as conn:
cursor = conn.cursor()
address = km.new_address(cursor)
response = {"id": message["id"], "result": address.serialize()}
syncer.queues[message['sender']].put(response)
if message['action'] == "get confirmed balance stats":
response = {"id": message["id"]}
try:
height = get_height()
with km.open() as conn:
cursor = conn.cursor()
stats = km.get_confirmed_balance_stats(height, cursor)
response["result"] = stats
except KeyError:
response[
"result"] = "error: core_loop didn't set height yet"
except Exception as e:
response["result"] = "error: " + str(e)
syncer.queues[message['sender']].put(response)
if message['action'] == "get confirmed balance list":
response = {"id": message["id"]}
try:
height = get_height()
with km.open() as conn:
cursor = conn.cursor()
stats = km.get_confirmed_balance_list(height, cursor)
response["result"] = stats
except KeyError:
response[
"result"] = "error: core_loop didn't set height yet"
except Exception as e:
response["result"] = "error: " + str(e)
syncer.queues[message['sender']].put(response)
if message['action'] == "give private key":
taddress = message["address"]
a = Address()
a.from_text(taddress)
with km.open() as conn:
cursor = conn.cursor()
priv = km.priv_by_address(a, cursor)
response = {"id": message["id"], "result": priv.private_key}
syncer.queues[message['sender']].put(response)
if message['action'] == "take private key":
response = {"id": message["id"]}
rescan = bool(message["rescan"])
ser_privkey = message["privkey"]
privkey = PrivateKey(ser_privkey, raw=True)
with km.open() as conn:
cursor = conn.cursor()
res = km.add_privkey(privkey, cursor, duplicate_safe=True)
if res and not rescan:
response["result"] = "success"
elif rescan:
response["result"] = "failed"
response["error"] = "rescan is not implemented"
else:
response["result"] = "failed"
syncer.queues[message['sender']].put(response)
continue
if message['action'] == "give last transactions info":
response = {"id": message["id"]}
num = int(message["num"])
with km.open() as conn:
cursor = conn.cursor()
response["result"] = km.give_transactions(num, cursor)
syncer.queues[message['sender']].put(response)
continue
if message['action'] == "generate tx":
response = {"id": message["id"]}
value = int(message["value"])
taddress = message["address"]
a = Address()
a.from_text(taddress)
try:
current_height = get_height()
except KeyError:
response["result"] = "error"
response["error"] = "core_loop didn't set height yet"
syncer.queues[message['sender']].put(response)
continue
except Exception as e:
response["result"] = "error"
response["error"] = str(e)
syncer.queues[message['sender']].put(response)
continue
with km.open() as conn:
cursor = conn.cursor()
_list = km.get_confirmed_balance_list(
current_height, cursor)
list_to_spend = []
summ = 0
utxos = []
expected_fee = 0
for address in _list:
for texted_index in _list[address]:
if summ > value + expected_fee: #TODO fee here
continue
if isinstance(_list[address][texted_index], int):
_index = base64.b64decode(
texted_index.encode())
ser_priv, ser_blinding, apc = km.get_output_private_data(
_index, cursor)
priv = PrivateKey(ser_priv, raw=True)
blinding = PrivateKey(ser_blinding, raw=True)
utxos.append(
(_index, _list[address][texted_index],
priv, blinding, apc))
summ += _list[address][texted_index]
expected_fee += 60000 #Should be enough
if summ < value:
response["result"] = "error"
response["error"] = "Not enough matured coins"
syncer.queues[message['sender']].put(response)
continue
if summ < value + expected_fee:
response["result"] = "error"
response[
"error"] = "Not enough matured coins for value and fees (%.8f)" % (
expected_fee / 1e8)
syncer.queues[message['sender']].put(response)
continue
tx = Transaction(None, None)
tx.add_destination((a, value, True))
tx.blindly_generate(
km.new_address(cursor), utxos,
config["fee_policy"].get("generate_fee_per_kb", 3000))
km.save_generated_transaction(tx, cursor)
response["result"] = tx.serialize()
syncer.queues[message['sender']].put(response)
if message['action'] == "stop":
logger.info("Wallet stops")
return