def core_loop(syncer, config):
init_storage_space(config)
nodes = {}
requests = {} # requests to other node's subprocesses
requests_cache = {
"blocks": [],
"txouts": []
} # requests of assets to other nodes
set_ask_for_blocks_hook(storage_space.blockchain, requests_cache)
set_ask_for_txouts_hook(storage_space.blocks_storage, requests_cache)
if config['wallet']:
set_notify_wallet_hook(storage_space.blockchain,
syncer.queues['Wallet'])
message_queue = syncer.queues['Blockchain']
message_queue.put({"action": "give nodes list reminder"})
message_queue.put({"action": "check requests cache"})
#set logging
default_log_level = logging.INFO
if "logging" in config: #debug, info, warning, error, critical
loglevels = {
"debug": logging.DEBUG,
"info": logging.INFO,
"warning": logging.WARNING,
"error": logging.ERROR,
"critical": logging.CRITICAL
}
if "base" in config["logging"] and config["logging"][
"base"] in loglevels:
logger.setLevel(loglevels[config["logging"]["base"]])
if "core" in config["logging"] and config["logging"][
"core"] in loglevels:
#its ok to rewrite
logger.setLevel(loglevels[config["logging"]["core"]])
is_benchmark = config.get('testnet_options', {}).get('benchmark', False)
no_pow = config.get('testnet_options', {}).get('do_not_check_pow', False)
def get_new_address(timeout=2.5): #blocking
_id = str(uuid4())
syncer.queues['Wallet'].put({
'action': 'give new address',
'id': _id,
'sender': "Blockchain"
})
result = None
start_time = time()
while True:
put_back = [
] #We wait for specific message, all others will wait for being processed
while not message_queue.empty():
message = message_queue.get()
if (not 'id' in message) or (not message['id'] == _id):
put_back.append(message)
continue
result = message['result']
break
for message in put_back:
message_queue.put(message)
if result:
break
sleep(0.01)
if time() - start_time > timeout:
raise Exception(
"get_new_address timeout: probably wallet has collapsed or not running"
)
if result == 'error':
raise Exception("Can not get_new_address: error on wallet side")
address = Address()
logger.info("Receiving address %s (len %d)" % (result, len(result)))
address.deserialize_raw(result)
return address
def send_message(destination, message):
logger.debug("Sending message to %s:\t\t %s" %
(str(destination), str(message)))
if not 'id' in message:
message['id'] = uuid4()
if not 'sender' in message:
message['sender'] = "Blockchain"
syncer.queues[destination].put(message)
def send_to_network(message):
send_message("NetworkManager", message)
notify = partial(set_value_to_queue, syncer.queues["Notifications"],
"Blockchain")
core_context = CoreContext(storage_space, logger, nodes, notify,
send_message, get_new_address, config)
logger.debug("Start of core loop")
with storage_space.env.begin(
write=True
) as rtx: #Set basic chain info, so wallet and other services can start work
notify("blockchain height",
storage_space.blockchain.current_height(rtx=rtx))
notify("best header", storage_space.headers_manager.best_header_height)
while True:
sleep(0.05)
put_back_messages = []
notify("core workload", "idle")
while not message_queue.empty():
message = message_queue.get()
if 'time' in message and message['time'] > time(
): # delay this message
put_back_messages.append(message)
continue
if (('result' in message) and message['result']=="processed") or \
(('result' in message) and message['result']=="set") or \
(('action' in message) and message['action']=="give nodes list reminder") or \
(('action' in message) and message['action']=="check requests cache") or \
(('action' in message) and message['action']=="take nodes list") or \
(('result' in message) and is_ip_port_array(message['result'])):
logger.debug("Processing message %s" % message)
else:
if 'action' in message:
logger.info("Processing message `%s`" % message['action'])
else:
logger.info("Processing message %s" % message)
if not 'action' in message: #it is response
if message['id'] in requests: # response is awaited
if requests[message['id']] == "give nodes list":
requests.pop(message['id'])
message_queue.put({
"action": "take nodes list",
"nodes": message["result"]
})
else:
pass #Drop
continue
try:
if ("node" in message) and (not message["node"] in nodes):
nodes[message["node"]] = {'node': message["node"]}
if message["action"] == "take the headers":
notify("core workload", "processing new headers")
with storage_space.env.begin(write=True) as wtx:
process_new_headers(message, nodes[message["node"]],
wtx, core_context)
notify("best header",
storage_space.headers_manager.best_header_height)
if message["action"] == "take the blocks":
notify("core workload", "processing new blocks")
with storage_space.env.begin(write=True) as wtx:
initial_tip = storage_space.blockchain.current_tip(
rtx=wtx)
process_new_blocks(message, wtx, core_context)
after_tip = storage_space.blockchain.current_tip(
rtx=wtx)
notify(
"blockchain height",
storage_space.blockchain.current_height(rtx=wtx))
if not after_tip == initial_tip:
notify_all_nodes_about_new_tip(nodes,
rtx=wtx,
core=core_context,
_except=[],
_payload_except=[])
look_forward(nodes, send_to_network, rtx=wtx)
if message["action"] == "take the txos":
notify("core workload", "processing new txos")
with storage_space.env.begin(write=True) as wtx:
process_new_txos(message, wtx=wtx, core=core_context)
#After downloading new txos some blocs may become downloaded
notify(
"blockchain height",
storage_space.blockchain.current_height(rtx=wtx))
look_forward(nodes, send_to_network, rtx=wtx)
if message[
"action"] in request_handlers: #blocks, headers, txos and tbm
notify("core workload", "processing " + message["action"])
with storage_space.env.begin(write=False) as rtx:
request_handlers[message["action"]](message,
rtx=rtx,
core=core_context)
if message[
"action"] in metadata_handlers: # take tip, find common root [response]
with storage_space.env.begin(write=False) as rtx:
metadata_handlers[message["action"]](
message,
nodes[message["node"]],
rtx=rtx,
core=core_context)
if message["action"] == "take TBM transaction":
notify("core workload", "processing mempool tx")
with storage_space.env.begin(write=False) as rtx:
process_tbm_tx(message, rtx=rtx, core=core_context)
if message["action"] == "give tip height":
with storage_space.env.begin(write=False) as rtx:
_ch = storage_space.blockchain.current_height(rtx=rtx)
send_message(message["sender"], {
"id": message["id"],
"result": _ch
})
notify("blockchain height", _ch)
except DOSException as e:
logger.info("DOS Exception %s" % str(e))
#raise e #TODO send to NM
except Exception as e:
raise e
if message["action"] == "give block info":
notify("core workload", "reading block info")
try:
with storage_space.env.begin(write=False) as rtx:
block_info = compose_block_info(message["block_num"],
rtx=rtx)
send_message(message["sender"], {
"id": message["id"],
"result": block_info
})
except Exception as e:
send_message(message["sender"], {
"id": message["id"],
"result": "error",
"error": str(e)
})
if message[
"action"] == "put arbitrary mining work" and is_benchmark:
if not no_pow:
raise Exception(
"`put arbitrary mining work` is only allowed for disabled pow checks"
)
notify("core workload", "putting arbitrary mining work")
message["nonce"] = b"\x00" * 8
message['partial_hash'] = list(
storage_space.mempool_tx.work_block_assoc.inner_dict.keys(
))[-1]
message['action'] = "take mining work"
if message[
"action"] in mining_operations: #getwork, gbt, submitblock, submitwork
notify("core workload", "processing" + message["action"])
with storage_space.env.begin(write=True) as wtx:
mining_operations[message["action"]](message, wtx,
core_context)
if message["action"] == "set mining address" and is_benchmark:
address = Address()
address.deserialize_raw(message["address"])
core_context.mining_address = address
if message["action"] == "give synchronization status":
with storage_space.env.begin(write=False) as rtx:
our_height = storage_space.blockchain.current_height(
rtx=rtx)
best_known_header = storage_space.headers_manager.best_header_height
try:
best_advertised_height = max([
nodes[node]["height"] for node in nodes
if "height" in nodes[node]
])
except:
best_advertised_height = None
send_message(
message["sender"], {
"id": message["id"],
"result": {
'height': our_height,
'best_known_header': best_known_header,
'best_advertised_height': best_advertised_height
}
})
notify("best header", best_known_header)
notify("blockchain height", our_height)
notify("best advertised height", best_advertised_height)
if message["action"] == "add tx to mempool":
notify("core workload", "processing local transaction")
response = {"id": message["id"]}
#deserialization
try:
ser_tx = message["tx"]
tx = Transaction(
txos_storage=storage_space.txos_storage,
excesses_storage=storage_space.excesses_storage)
with storage_space.env.begin(write=False) as rtx:
tx.deserialize(ser_tx, rtx)
storage_space.mempool_tx.add_tx(tx, rtx=rtx)
tx_skel = TransactionSkeleton(tx=tx)
notify_all_nodes_about_tx(tx_skel.serialize(
rich_format=True, max_size=40000),
core_context,
_except=[],
mode=1)
response['result'] = "generated"
except Exception as e:
response['result'] = 'error'
response['error'] = str(e)
logger.error("Problem in tx: %s" % str(e))
send_message(message["sender"], response)
#message from core_loop
if message[
"action"] in download_status_checks: # txouts and blocks download status checks
with storage_space.env.begin(write=True) as rtx:
ret_mes = download_status_checks[message["action"]](
message, rtx, core_context)
if ret_mes:
put_back_messages.append(ret_mes)
if message["action"] == "take nodes list":
for node in message["nodes"]:
if not node in nodes: #Do not overwrite
nodes[node] = {"node": node}
disconnected_nodes = []
for existing_node in nodes:
if not existing_node in message["nodes"]:
disconnected_nodes.append(existing_node)
for dn in disconnected_nodes:
nodes.pop(dn)
if message["action"] == "give nodes list reminder":
_id = str(uuid4())
send_to_network({
"action": "give intrinsic nodes list",
"sender": "Blockchain",
"id": _id
})
requests[_id] = "give nodes list"
put_back_messages.append({
"action": "give nodes list reminder",
"time": int(time()) + 3
})
if message["action"] == "stop":
logger.info("Core loop stops")
return
if message["action"] == "shutdown":
initiator = message["sender"]
logger.info("Shutdown initiated by %s" % initiator)
for receiver in [
'NetworkManager', 'Blockchain', 'RPCManager',
'Notifications', 'Wallet'
]:
send_message(receiver, {
"action": "stop",
"sender": initiator
})
if message["action"] == "check requests cache":
put_back_messages.append({
"action": "check requests cache",
"time": int(time()) + 5
})
for k in requests_cache:
if not len(requests_cache[k]):
continue
copy = list(set(requests_cache[k]))
copy = sorted(copy,
key=lambda x: requests_cache[k].index(x)
) #preserve order of downloaded objects
if k == "blocks":
chunk_size = 20
while len(copy):
request, copy = copy[:chunk_size], copy[
chunk_size:]
new_message = {
"action": "check blocks download status",
"block_hashes": request,
"already_asked_nodes": [],
"id": str(uuid4()),
"time": -1
}
message_queue.put(new_message)
requests_cache[k] = []
if k == "txouts":
chunk_size = 30
while len(copy):
request, copy = copy[:chunk_size], copy[
chunk_size:]
new_message = {
"action": "check txouts download status",
"txos_hashes": request,
"already_asked_nodes": [],
"id": str(uuid4()),
"time": -1
}
message_queue.put(new_message)
requests_cache[k] = []
for _message in put_back_messages:
message_queue.put(_message)
try:
with storage_space.env.begin(write=True) as rtx:
check_sync_status(nodes, rtx=rtx, core_context=core_context)
try:
best_advertised_height = max([
nodes[node]["height"] for node in nodes
if "height" in nodes[node]
])
except:
best_advertised_height = None
notify("best advertised height", best_advertised_height)
except Exception as e:
logger.error(e)
def wallet(syncer, config):
'''
Wallet is synchronous service which holds private keys and information about
owned outputs. It provides information for transactions and block templates
generation.
'''
def get_height(timeout=2.5):
_id = str(uuid4())
syncer.queues['Notifications'].put({
'action': 'get',
'id': _id,
'key': 'blockchain height',
'sender': "Wallet"
})
message_queue = syncer.queues['Wallet']
start_time = time()
result = None
while True:
put_back = [
] #We wait for specific message, all others will wait for being processed
while not message_queue.empty():
message = message_queue.get()
if (not 'id' in message) or (not message['id'] == _id):
put_back.append(message)
continue
result = message['result']
break
for message in put_back:
message_queue.put(message)
if result:
break
sleep(0.01)
if time() - start_time > timeout:
raise KeyError
if result == 'error':
raise KeyError
return result['value']
notification_cache = {}
def notify(key, value, timestamp=None):
if (key in notification_cache) and (
notification_cache[key]['value'] == value
) and (time() - notification_cache[key]['timestamp']) < 5:
return #Do not spam notifications with the same values
message = {}
message['id'] = uuid4()
message['sender'] = "Wallet"
if not timestamp:
timestamp = time()
message['time'] = timestamp
message['action'] = "set"
message['key'] = key
message['value'] = value
syncer.queues["Notifications"].put(message)
notification_cache[key] = {'value': value, 'timestamp': timestamp}
#set logging
default_log_level = logging.INFO
if "logging" in config: #debug, info, warning, error, critical
loglevels = {
"debug": logging.DEBUG,
"info": logging.INFO,
"warning": logging.WARNING,
"error": logging.ERROR,
"critical": logging.CRITICAL
}
if "base" in config["logging"] and config["logging"][
"base"] in loglevels:
logger.setLevel(loglevels[config["logging"]["base"]])
if "wallet" in config["logging"] and config["logging"][
"wallet"] in loglevels:
#its ok to rewrite
logger.setLevel(loglevels[config["logging"]["wallet"]])
message_queue = syncer.queues['Wallet']
_path = config['location']['wallet']
try:
password = config['wallet'].get('password', None)
except:
password = None
km = KeyDB(path=_path, password=password)
with km.open() as conn:
cursor = conn.cursor()
apply_migrations(cursor)
notify('last wallet update', time())
while True:
sleep(0.01)
while not message_queue.empty():
message = message_queue.get()
if 'action' in message:
logger.info("Process message `%s`" % message['action'])
logger.debug("Process message %s" % message)
else:
logger.info("Process message %s" % message)
if not 'action' in message:
continue
if message['action'] == "process new block":
tx = Transaction(txos_storage=None, excesses_storage=None)
tx.deserialize(
message['tx'], rtx=None, skip_verification=True
) #skip_verification allows us to not provide rtx
block_height = message['height']
last_time_updated = None
with km.open() as conn:
cursor = conn.cursor()
for index in tx.inputs:
#Note it is not check whether output is unspent or not, we check that output is marked as our and unspent in our wallet
if km.is_unspent(index, cursor):
km.spend_output(index, block_height, cursor)
last_time_updated = time()
for _o in tx.outputs:
if km.is_owned_pubkey(_o.address.pubkey.serialize(),
cursor):
km.add_output(_o, block_height, cursor)
last_time_updated = time()
if km.is_saved(_o, cursor):
km.register_processed_output(
_o.serialized_index, block_height, cursor)
if last_time_updated:
notify('last wallet update', last_time_updated)
if message['action'] == "process rollback":
rollback = message['rollback_object']
block_height = message['block_height']
with km.open() as conn:
cursor = conn.cursor()
km.rollback(block_height, cursor)
last_time_updated = time()
notify('last wallet update', last_time_updated)
if message[
'action'] == "process indexed outputs": #during private key import correspondent outputs will be processed again
pass
if message['action'] == "give new taddress":
with km.open() as conn:
cursor = conn.cursor()
address = km.new_address(cursor)
response = {"id": message["id"], "result": address.to_text()}
syncer.queues[message['sender']].put(response)
if message['action'] == "give new address":
with km.open() as conn:
cursor = conn.cursor()
address = km.new_address(cursor)
response = {"id": message["id"], "result": address.serialize()}
syncer.queues[message['sender']].put(response)
if message['action'] == "get confirmed balance stats":
response = {"id": message["id"]}
try:
height = get_height()
with km.open() as conn:
cursor = conn.cursor()
stats = km.get_confirmed_balance_stats(height, cursor)
response["result"] = stats
except KeyError:
response[
"result"] = "error: core_loop didn't set height yet"
except Exception as e:
response["result"] = "error: " + str(e)
syncer.queues[message['sender']].put(response)
if message['action'] == "get confirmed balance list":
response = {"id": message["id"]}
try:
height = get_height()
with km.open() as conn:
cursor = conn.cursor()
stats = km.get_confirmed_balance_list(height, cursor)
response["result"] = stats
except KeyError:
response[
"result"] = "error: core_loop didn't set height yet"
except Exception as e:
response["result"] = "error: " + str(e)
syncer.queues[message['sender']].put(response)
if message['action'] == "give private key":
taddress = message["address"]
a = Address()
a.from_text(taddress)
with km.open() as conn:
cursor = conn.cursor()
priv = km.priv_by_address(a, cursor)
response = {"id": message["id"], "result": priv.private_key}
syncer.queues[message['sender']].put(response)
if message['action'] == "take private key":
response = {"id": message["id"]}
rescan = bool(message["rescan"])
ser_privkey = message["privkey"]
privkey = PrivateKey(ser_privkey, raw=True)
with km.open() as conn:
cursor = conn.cursor()
res = km.add_privkey(privkey, cursor, duplicate_safe=True)
if res and not rescan:
response["result"] = "success"
elif rescan:
response["result"] = "failed"
response["error"] = "rescan is not implemented"
else:
response["result"] = "failed"
syncer.queues[message['sender']].put(response)
continue
if message['action'] == "give last transactions info":
response = {"id": message["id"]}
num = int(message["num"])
with km.open() as conn:
cursor = conn.cursor()
response["result"] = km.give_transactions(num, cursor)
syncer.queues[message['sender']].put(response)
continue
if message['action'] == "generate tx":
response = {"id": message["id"]}
value = int(message["value"])
taddress = message["address"]
a = Address()
a.from_text(taddress)
try:
current_height = get_height()
except KeyError:
response["result"] = "error"
response["error"] = "core_loop didn't set height yet"
syncer.queues[message['sender']].put(response)
continue
except Exception as e:
response["result"] = "error"
response["error"] = str(e)
syncer.queues[message['sender']].put(response)
continue
with km.open() as conn:
cursor = conn.cursor()
_list = km.get_confirmed_balance_list(
current_height, cursor)
list_to_spend = []
summ = 0
utxos = []
expected_fee = 0
for address in _list:
for texted_index in _list[address]:
if summ > value + expected_fee: #TODO fee here
continue
if isinstance(_list[address][texted_index], int):
_index = base64.b64decode(
texted_index.encode())
ser_priv, ser_blinding, apc = km.get_output_private_data(
_index, cursor)
priv = PrivateKey(ser_priv, raw=True)
blinding = PrivateKey(ser_blinding, raw=True)
utxos.append(
(_index, _list[address][texted_index],
priv, blinding, apc))
summ += _list[address][texted_index]
expected_fee += 60000 #Should be enough
if summ < value:
response["result"] = "error"
response["error"] = "Not enough matured coins"
syncer.queues[message['sender']].put(response)
continue
if summ < value + expected_fee:
response["result"] = "error"
response[
"error"] = "Not enough matured coins for value and fees (%.8f)" % (
expected_fee / 1e8)
syncer.queues[message['sender']].put(response)
continue
tx = Transaction(None, None)
tx.add_destination((a, value, True))
tx.blindly_generate(
km.new_address(cursor), utxos,
config["fee_policy"].get("generate_fee_per_kb", 3000))
km.save_generated_transaction(tx, cursor)
response["result"] = tx.serialize()
syncer.queues[message['sender']].put(response)
if message['action'] == "stop":
logger.info("Wallet stops")
return