def update_current_set(self):
'''
For now we have quite a simple and dirty algo:
1) sort all tx by input_num (bigger first)
2) iterate through tx and add transactions to current_set if
a) it is downloaded
b) it is valid (otherwise delete from self.transactions)
c) doesn't contradict with any other tx in the set
'''
self.transactions = sorted(self.transactions,
key=lambda x: len(x.input_indexes),
reverse=True)
tx_to_remove_list = []
txos_storage = self.storage_space.txos_storage
merged_tx = Transaction(txos_storage=txos_storage)
for tx_skeleton in self.transactions:
#TODO build_tx_from_skeleton should raise distinctive exceptions
downloaded = True
for _i in tx_skeleton.input_indexes:
if not _i in txos_storage.confirmed:
downloaded = False
for _o in tx_skeleton.output_indexes:
if not _o in txos_storage.mempool:
downloaded = False
if not downloaded:
continue
try:
if tx_skeleton.serialize() in self.built_tx:
full_tx = self.built_tx[tx_skeleton.serialize()]
else:
if tx_skeleton.tx:
full_tx = tx_skeleton.tx
else:
full_tx = build_tx_from_skeleton(
tx_skeleton, self.storage_space.txos_storage,
self.storage_space.blockchain.current_height + 1)
tx_skeleton.tx = full_tx
self.built_tx[tx_skeleton.serialize()] = full_tx
except Exception as e:
tx_to_remove_list.append(tx_skeleton)
continue
try:
if self.fee_policy_checker.check_tx(full_tx):
merged_tx = merged_tx.merge(full_tx)
self.current_set.append(tx_skeleton)
except Exception as e:
pass #it is ok, tx contradicts with other transactions in the pool
for tx in tx_to_remove_list:
self.transactions.remove(tx)
self.built_tx.pop(tx.serialize(), None)
self.combined_tx = merged_tx
def build_tx_from_skeleton(tx_skeleton,
txos_storage,
block_height,
historical=False,
non_context=False):
'''
By given tx_skeleton and txos_storage return transaction.
If transaction is invalid or any input/output isn't available exception will be raised.
Optionally, if `historical` is True we will check output_indexes both in mempool and spent outputs.
'''
tx = Transaction(txos_storage=txos_storage)
for _i in tx_skeleton.input_indexes:
tx.inputs.append(txos_storage.confirmed[_i])
for _o in tx_skeleton.output_indexes:
if historical:
try:
tx.outputs.append(txos_storage.confirmed.find(_o))
except:
tx.outputs.append(txos_storage.mempool[_o])
else:
tx.outputs.append(txos_storage.mempool[_o])
tx.additional_excesses = tx_skeleton.additional_excesses.copy()
tx.combined_excesses = tx_skeleton.combined_excesses.copy()
if historical or non_context:
assert tx.non_context_verify(block_height=block_height)
else:
assert tx.verify(block_height=block_height)
return tx
def give_block_template(self):
if not self.key_manager:
raise Exception("Key manager is not set")
transaction_fees = self.give_tx().relay_fee if self.give_tx() else 0
value = next_reward(
self.storage_space.blockchain.current_tip,
self.storage_space.headers_storage) + transaction_fees
coinbase = IOput()
coinbase.fill(
self.key_manager.new_address(),
value,
relay_fee=0,
coinbase=True,
lock_height=self.storage_space.blockchain.current_height + 1 +
coinbase_maturity)
coinbase.generate()
self.storage_space.txos_storage.mempool[
coinbase.serialized_index] = coinbase
tx = Transaction(txos_storage=self.storage_space.txos_storage,
key_manager=self.key_manager)
tx.add_coinbase(coinbase)
tx.compose_block_transaction()
block = generate_block_template(tx, self.storage_space)
self.add_block_template(block)
return block
def build_tx_from_skeleton(tx_skeleton,
txos_storage,
excesses_storage,
block_height,
block_version,
rtx,
historical=False,
non_context=False):
'''
By given tx_skeleton and txos_storage return transaction.
If transaction is invalid or any input/output isn't available exception will be raised.
Optionally, if `historical` is True we will check output_indexes both in mempool and spent outputs.
'''
tx = Transaction(txos_storage=txos_storage,
excesses_storage=excesses_storage)
for _i in tx_skeleton.input_indexes:
if historical or non_context:
tx.inputs.append(txos_storage.confirmed.find(_i, rtx=rtx))
else:
tx.inputs.append(txos_storage.confirmed.get(_i, rtx=rtx))
for _o in tx_skeleton.output_indexes:
if historical or non_context:
# About non_context: if we are on one branch and build block from another one
# and this block contain output which is already commited on our branch (tx is
# confirmed on both branches) we should get txo from confirmed storage
try:
tx.outputs.append(txos_storage.confirmed.find(_o, rtx=rtx))
except:
tx.outputs.append(txos_storage.mempool[_o])
else:
tx.outputs.append(txos_storage.mempool[_o])
tx.additional_excesses = tx_skeleton.additional_excesses.copy()
tx.updated_excesses = tx_skeleton.updated_excesses.copy()
tx.mixer_offset = tx_skeleton.mixer_offset
if historical or non_context:
assert tx.non_context_verify(block_height=block_height)
else:
assert tx.verify(block_height=block_height,
block_version=block_version,
rtx=rtx)
return tx
def core_loop(syncer, config):
init_storage_space(config)
nodes = {}
requests = {} # requests to other node's subprocesses
requests_cache = {
"blocks": [],
"txouts": []
} # requests of assets to other nodes
set_ask_for_blocks_hook(storage_space.blockchain, requests_cache)
set_ask_for_txouts_hook(storage_space.blocks_storage, requests_cache)
if config['wallet']:
set_notify_wallet_hook(storage_space.blockchain,
syncer.queues['Wallet'])
message_queue = syncer.queues['Blockchain']
message_queue.put({"action": "give nodes list reminder"})
message_queue.put({"action": "check requests cache"})
#set logging
default_log_level = logging.INFO
if "logging" in config: #debug, info, warning, error, critical
loglevels = {
"debug": logging.DEBUG,
"info": logging.INFO,
"warning": logging.WARNING,
"error": logging.ERROR,
"critical": logging.CRITICAL
}
if "base" in config["logging"] and config["logging"][
"base"] in loglevels:
logger.setLevel(loglevels[config["logging"]["base"]])
if "core" in config["logging"] and config["logging"][
"core"] in loglevels:
#its ok to rewrite
logger.setLevel(loglevels[config["logging"]["core"]])
is_benchmark = config.get('testnet_options', {}).get('benchmark', False)
no_pow = config.get('testnet_options', {}).get('do_not_check_pow', False)
def get_new_address(timeout=2.5): #blocking
_id = str(uuid4())
syncer.queues['Wallet'].put({
'action': 'give new address',
'id': _id,
'sender': "Blockchain"
})
result = None
start_time = time()
while True:
put_back = [
] #We wait for specific message, all others will wait for being processed
while not message_queue.empty():
message = message_queue.get()
if (not 'id' in message) or (not message['id'] == _id):
put_back.append(message)
continue
result = message['result']
break
for message in put_back:
message_queue.put(message)
if result:
break
sleep(0.01)
if time() - start_time > timeout:
raise Exception(
"get_new_address timeout: probably wallet has collapsed or not running"
)
if result == 'error':
raise Exception("Can not get_new_address: error on wallet side")
address = Address()
logger.info("Receiving address %s (len %d)" % (result, len(result)))
address.deserialize_raw(result)
return address
def send_message(destination, message):
logger.debug("Sending message to %s:\t\t %s" %
(str(destination), str(message)))
if not 'id' in message:
message['id'] = uuid4()
if not 'sender' in message:
message['sender'] = "Blockchain"
syncer.queues[destination].put(message)
def send_to_network(message):
send_message("NetworkManager", message)
notify = partial(set_value_to_queue, syncer.queues["Notifications"],
"Blockchain")
core_context = CoreContext(storage_space, logger, nodes, notify,
send_message, get_new_address, config)
logger.debug("Start of core loop")
with storage_space.env.begin(
write=True
) as rtx: #Set basic chain info, so wallet and other services can start work
notify("blockchain height",
storage_space.blockchain.current_height(rtx=rtx))
notify("best header", storage_space.headers_manager.best_header_height)
while True:
sleep(0.05)
put_back_messages = []
notify("core workload", "idle")
while not message_queue.empty():
message = message_queue.get()
if 'time' in message and message['time'] > time(
): # delay this message
put_back_messages.append(message)
continue
if (('result' in message) and message['result']=="processed") or \
(('result' in message) and message['result']=="set") or \
(('action' in message) and message['action']=="give nodes list reminder") or \
(('action' in message) and message['action']=="check requests cache") or \
(('action' in message) and message['action']=="take nodes list") or \
(('result' in message) and is_ip_port_array(message['result'])):
logger.debug("Processing message %s" % message)
else:
if 'action' in message:
logger.info("Processing message `%s`" % message['action'])
else:
logger.info("Processing message %s" % message)
if not 'action' in message: #it is response
if message['id'] in requests: # response is awaited
if requests[message['id']] == "give nodes list":
requests.pop(message['id'])
message_queue.put({
"action": "take nodes list",
"nodes": message["result"]
})
else:
pass #Drop
continue
try:
if ("node" in message) and (not message["node"] in nodes):
nodes[message["node"]] = {'node': message["node"]}
if message["action"] == "take the headers":
notify("core workload", "processing new headers")
with storage_space.env.begin(write=True) as wtx:
process_new_headers(message, nodes[message["node"]],
wtx, core_context)
notify("best header",
storage_space.headers_manager.best_header_height)
if message["action"] == "take the blocks":
notify("core workload", "processing new blocks")
with storage_space.env.begin(write=True) as wtx:
initial_tip = storage_space.blockchain.current_tip(
rtx=wtx)
process_new_blocks(message, wtx, core_context)
after_tip = storage_space.blockchain.current_tip(
rtx=wtx)
notify(
"blockchain height",
storage_space.blockchain.current_height(rtx=wtx))
if not after_tip == initial_tip:
notify_all_nodes_about_new_tip(nodes,
rtx=wtx,
core=core_context,
_except=[],
_payload_except=[])
look_forward(nodes, send_to_network, rtx=wtx)
if message["action"] == "take the txos":
notify("core workload", "processing new txos")
with storage_space.env.begin(write=True) as wtx:
process_new_txos(message, wtx=wtx, core=core_context)
#After downloading new txos some blocs may become downloaded
notify(
"blockchain height",
storage_space.blockchain.current_height(rtx=wtx))
look_forward(nodes, send_to_network, rtx=wtx)
if message[
"action"] in request_handlers: #blocks, headers, txos and tbm
notify("core workload", "processing " + message["action"])
with storage_space.env.begin(write=False) as rtx:
request_handlers[message["action"]](message,
rtx=rtx,
core=core_context)
if message[
"action"] in metadata_handlers: # take tip, find common root [response]
with storage_space.env.begin(write=False) as rtx:
metadata_handlers[message["action"]](
message,
nodes[message["node"]],
rtx=rtx,
core=core_context)
if message["action"] == "take TBM transaction":
notify("core workload", "processing mempool tx")
with storage_space.env.begin(write=False) as rtx:
process_tbm_tx(message, rtx=rtx, core=core_context)
if message["action"] == "give tip height":
with storage_space.env.begin(write=False) as rtx:
_ch = storage_space.blockchain.current_height(rtx=rtx)
send_message(message["sender"], {
"id": message["id"],
"result": _ch
})
notify("blockchain height", _ch)
except DOSException as e:
logger.info("DOS Exception %s" % str(e))
#raise e #TODO send to NM
except Exception as e:
raise e
if message["action"] == "give block info":
notify("core workload", "reading block info")
try:
with storage_space.env.begin(write=False) as rtx:
block_info = compose_block_info(message["block_num"],
rtx=rtx)
send_message(message["sender"], {
"id": message["id"],
"result": block_info
})
except Exception as e:
send_message(message["sender"], {
"id": message["id"],
"result": "error",
"error": str(e)
})
if message[
"action"] == "put arbitrary mining work" and is_benchmark:
if not no_pow:
raise Exception(
"`put arbitrary mining work` is only allowed for disabled pow checks"
)
notify("core workload", "putting arbitrary mining work")
message["nonce"] = b"\x00" * 8
message['partial_hash'] = list(
storage_space.mempool_tx.work_block_assoc.inner_dict.keys(
))[-1]
message['action'] = "take mining work"
if message[
"action"] in mining_operations: #getwork, gbt, submitblock, submitwork
notify("core workload", "processing" + message["action"])
with storage_space.env.begin(write=True) as wtx:
mining_operations[message["action"]](message, wtx,
core_context)
if message["action"] == "set mining address" and is_benchmark:
address = Address()
address.deserialize_raw(message["address"])
core_context.mining_address = address
if message["action"] == "give synchronization status":
with storage_space.env.begin(write=False) as rtx:
our_height = storage_space.blockchain.current_height(
rtx=rtx)
best_known_header = storage_space.headers_manager.best_header_height
try:
best_advertised_height = max([
nodes[node]["height"] for node in nodes
if "height" in nodes[node]
])
except:
best_advertised_height = None
send_message(
message["sender"], {
"id": message["id"],
"result": {
'height': our_height,
'best_known_header': best_known_header,
'best_advertised_height': best_advertised_height
}
})
notify("best header", best_known_header)
notify("blockchain height", our_height)
notify("best advertised height", best_advertised_height)
if message["action"] == "add tx to mempool":
notify("core workload", "processing local transaction")
response = {"id": message["id"]}
#deserialization
try:
ser_tx = message["tx"]
tx = Transaction(
txos_storage=storage_space.txos_storage,
excesses_storage=storage_space.excesses_storage)
with storage_space.env.begin(write=False) as rtx:
tx.deserialize(ser_tx, rtx)
storage_space.mempool_tx.add_tx(tx, rtx=rtx)
tx_skel = TransactionSkeleton(tx=tx)
notify_all_nodes_about_tx(tx_skel.serialize(
rich_format=True, max_size=40000),
core_context,
_except=[],
mode=1)
response['result'] = "generated"
except Exception as e:
response['result'] = 'error'
response['error'] = str(e)
logger.error("Problem in tx: %s" % str(e))
send_message(message["sender"], response)
#message from core_loop
if message[
"action"] in download_status_checks: # txouts and blocks download status checks
with storage_space.env.begin(write=True) as rtx:
ret_mes = download_status_checks[message["action"]](
message, rtx, core_context)
if ret_mes:
put_back_messages.append(ret_mes)
if message["action"] == "take nodes list":
for node in message["nodes"]:
if not node in nodes: #Do not overwrite
nodes[node] = {"node": node}
disconnected_nodes = []
for existing_node in nodes:
if not existing_node in message["nodes"]:
disconnected_nodes.append(existing_node)
for dn in disconnected_nodes:
nodes.pop(dn)
if message["action"] == "give nodes list reminder":
_id = str(uuid4())
send_to_network({
"action": "give intrinsic nodes list",
"sender": "Blockchain",
"id": _id
})
requests[_id] = "give nodes list"
put_back_messages.append({
"action": "give nodes list reminder",
"time": int(time()) + 3
})
if message["action"] == "stop":
logger.info("Core loop stops")
return
if message["action"] == "shutdown":
initiator = message["sender"]
logger.info("Shutdown initiated by %s" % initiator)
for receiver in [
'NetworkManager', 'Blockchain', 'RPCManager',
'Notifications', 'Wallet'
]:
send_message(receiver, {
"action": "stop",
"sender": initiator
})
if message["action"] == "check requests cache":
put_back_messages.append({
"action": "check requests cache",
"time": int(time()) + 5
})
for k in requests_cache:
if not len(requests_cache[k]):
continue
copy = list(set(requests_cache[k]))
copy = sorted(copy,
key=lambda x: requests_cache[k].index(x)
) #preserve order of downloaded objects
if k == "blocks":
chunk_size = 20
while len(copy):
request, copy = copy[:chunk_size], copy[
chunk_size:]
new_message = {
"action": "check blocks download status",
"block_hashes": request,
"already_asked_nodes": [],
"id": str(uuid4()),
"time": -1
}
message_queue.put(new_message)
requests_cache[k] = []
if k == "txouts":
chunk_size = 30
while len(copy):
request, copy = copy[:chunk_size], copy[
chunk_size:]
new_message = {
"action": "check txouts download status",
"txos_hashes": request,
"already_asked_nodes": [],
"id": str(uuid4()),
"time": -1
}
message_queue.put(new_message)
requests_cache[k] = []
for _message in put_back_messages:
message_queue.put(_message)
try:
with storage_space.env.begin(write=True) as rtx:
check_sync_status(nodes, rtx=rtx, core_context=core_context)
try:
best_advertised_height = max([
nodes[node]["height"] for node in nodes
if "height" in nodes[node]
])
except:
best_advertised_height = None
notify("best advertised height", best_advertised_height)
except Exception as e:
logger.error(e)
def core_loop(syncer, config):
message_queue = syncer.queues['Blockchain']
init_storage_space(config)
nodes = {}
set_ask_for_blocks_hook(storage_space.blockchain, message_queue)
set_ask_for_txouts_hook(storage_space.blocks_storage, message_queue)
requests = {}
message_queue.put({"action": "give nodes list reminder"})
def send_message(destination, message):
if not 'id' in message:
message['id'] = uuid4()
if not 'sender' in message:
message['sender'] = "Blockchain"
syncer.queues[destination].put(message)
def send_to_nm(message):
send_message("NetworkManager", message)
logger.debug("Start of core loop")
while True:
sleep(0.05)
put_back_messages = []
while not message_queue.empty():
message = message_queue.get()
if 'time' in message and message['time'] > time(
): # delay this message
put_back_messages.append(message)
continue
logger.info("Processing message %s" % message)
if not 'action' in message: #it is response
if message['id'] in requests: # response is awaited
if requests[message['id']] == "give nodes list":
requests.pop(message['id'])
message_queue.put({
"action": "take nodes list",
"nodes": message["result"]
})
else:
pass #Drop
continue
try:
if message["action"] == "take the headers":
process_new_headers(message)
if message["action"] == "take the blocks":
initial_tip = storage_space.blockchain.current_tip
process_new_blocks(message)
after_tip = storage_space.blockchain.current_tip
if not after_tip == initial_tip:
notify_all_nodes_about_new_tip(nodes, send_to_nm)
if message["action"] == "take the txos":
process_new_txos(message)
if message["action"] == "give blocks":
process_blocks_request(message, send_message)
if message["action"] == "give next headers":
process_next_headers_request(message, send_message)
if message["action"] == "give txos":
process_txos_request(message, send_message)
if message["action"] == "find common root":
process_find_common_root(message, send_message)
if message["action"] == "find common root response":
process_find_common_root_reponse(message,
nodes[message["node"]],
send_message)
if message["action"] == "give TBM transaction":
process_tbm_tx_request(message, send_message)
if message["action"] == "take TBM transaction":
process_tbm_tx(message, send_to_nm, nodes)
if message["action"] == "give tip height":
send_message(
message["sender"], {
"id": message["id"],
"result": storage_space.blockchain.current_height
})
if message["action"] == "take tip info":
if not message["node"] in nodes:
nodes[message["node"]] = {'node': message["node"]}
process_tip_info(message,
nodes[message["node"]],
send=send_to_nm)
except DOSException as e:
logger.info("DOS Exception %s" % str(e))
#raise e #TODO send to NM
except Exception as e:
raise e
if message["action"] == "give block template":
block = storage_space.mempool_tx.give_block_template()
ser_head = block.header.serialize()
send_message(message["sender"], {
"id": message["id"],
"result": ser_head
})
if message["action"] == "take solved block template":
try:
initial_tip = storage_space.blockchain.current_tip
header = Header()
header.deserialize(message["solved template"])
solved_block = storage_space.mempool_tx.get_block_by_header_solution(
header)
storage_space.headers_manager.add_header(
solved_block.header)
storage_space.headers_manager.context_validation(
solved_block.header.hash)
solved_block.non_context_verify()
storage_space.blockchain.add_block(solved_block)
send_message(message["sender"], {
"id": message["id"],
"result": "Accepted"
})
after_tip = storage_space.blockchain.current_tip
if not after_tip == initial_tip:
notify_all_nodes_about_new_tip(nodes, send_to_nm)
except Exception as e:
raise e
send_message(message["sender"], {
"id": message["id"],
"error": str(e)
})
if message["action"] == "get confirmed balance stats":
if storage_space.mempool_tx.key_manager:
stats = storage_space.mempool_tx.key_manager.get_confirmed_balance_stats(
storage_space.utxo_index, storage_space.txos_storage,
storage_space.blockchain.current_height)
send_message(message["sender"], {
"id": message["id"],
"result": stats
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "get confirmed balance list":
if storage_space.mempool_tx.key_manager:
_list = storage_space.mempool_tx.key_manager.get_confirmed_balance_list(
storage_space.utxo_index, storage_space.txos_storage,
storage_space.blockchain.current_height)
send_message(message["sender"], {
"id": message["id"],
"result": _list
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "give new address":
if storage_space.mempool_tx.key_manager:
texted_address = storage_space.mempool_tx.key_manager.new_address(
).to_text()
send_message(message["sender"], {
"id": message["id"],
"result": texted_address
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "give private key":
if storage_space.mempool_tx.key_manager:
km = storage_space.mempool_tx.key_manager
a = Address()
a.from_text(message["address"])
serialized_pk = km.priv_by_address(a).serialize()
send_message(message["sender"], {
"id": message["id"],
"result": serialized_pk
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "take private key":
if storage_space.mempool_tx.key_manager:
km = storage_space.mempool_tx.key_manager
pk = PrivateKey()
pk.deserialize(message['privkey'])
km.add_privkey(pk)
send_message(message["sender"], {
"id": message["id"],
"result": "imported"
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
if message["action"] == "give synchronization status":
our_height = storage_space.blockchain.current_height
best_known_header = storage_space.headers_manager.best_header_height
try:
best_advertised_height = max([
nodes[node]["height"] for node in nodes
if "height" in nodes[node]
])
except:
best_advertised_height = None
send_message(
message["sender"], {
"id": message["id"],
"result": {
'height': our_height,
'best_known_header': best_known_header,
'best_advertised_height': best_advertised_height
}
})
if message["action"] == "send to address":
value = int(message["value"])
taddress = message["address"]
a = Address()
a.from_text(taddress)
if storage_space.mempool_tx.key_manager:
_list = storage_space.mempool_tx.key_manager.get_confirmed_balance_list(
storage_space.utxo_index, storage_space.txos_storage,
storage_space.blockchain.current_height)
list_to_spend = []
summ = 0
for address in _list:
for texted_index in _list[address]:
if summ > value:
continue
if isinstance(_list[address][texted_index], int):
_index = base64.b64decode(
texted_index.encode())
utxo = storage_space.txos_storage.confirmed[
_index]
if not utxo.lock_height <= storage_space.blockchain.current_height:
continue
list_to_spend.append(utxo)
summ += _list[address][texted_index]
if summ < value:
send_message(
message["sender"], {
"id": message["id"],
"error": "Not enough matured coins"
})
tx = Transaction(
txos_storage=storage_space.txos_storage,
key_manager=storage_space.mempool_tx.key_manager)
for utxo in list_to_spend:
tx.push_input(utxo)
tx.add_destination((a, value))
tx.generate()
tx.verify()
storage_space.mempool_tx.add_tx(tx)
tx_skel = TransactionSkeleton(tx=tx)
notify_all_nodes_about_tx(tx_skel.serialize(
rich_format=True, max_size=40000),
nodes,
send_to_nm,
_except=[],
mode=1)
send_message(message["sender"], {
"id": message["id"],
"result": "generated"
})
else:
send_message(message["sender"], {
"id": message["id"],
"error": "No registered key manager"
})
#message from core_loop
if message["action"] == "check txouts download status":
txos = message["txos_hashes"]
to_be_downloaded = []
for txo in txos:
if not storage_space.txos_storage.known(txo):
to_be_downloaded.append(txo)
if not to_be_downloaded:
continue #We are good, txouts are already downloaded
already_asked_nodes = message["already_asked_nodes"]
asked = False
for node_params in nodes:
node = nodes[node_params]
if node in already_asked_nodes:
continue
already_asked_nodes += [node]
send_to_nm({
"action": "give txos",
"txos_hashes": b"".join(to_be_downloaded),
"num": len(to_be_downloaded),
"id": str(uuid4()),
"node": node_params
})
new_message = {
"action": "check txouts download status",
"txos_hashes": to_be_downloaded,
"already_asked_nodes": already_asked_nodes,
"id": str(uuid4()),
"time": int(time() + 300)
}
asked = True
put_back_messages.append(new_message)
break
if not asked: #We already asked all applicable nodes
message["time"] = int(time()) + 3600
message["already_asked_nodes"] = []
put_back_messages.append(
message) # we will try to ask again in an hour
#message from core_loop
if message["action"] == "check blocks download status":
#TODO download many blocks at once
block_hashes = message["block_hashes"]
to_be_downloaded = []
lowest_height = 1e10
for block_hash in block_hashes:
if block_hash in storage_space.blocks_storage:
continue #We are good, block already downloaded
if not block_hash in storage_space.blockchain.awaited_blocks:
continue #For some reason we don't need this block anymore
to_be_downloaded.append(block_hash)
if storage_space.headers_storage[
block_hash].height < lowest_height:
lowest_height = storage_space.headers_storage[
block_hash].height
already_asked_nodes = message["already_asked_nodes"]
asked = False
for node_params in nodes:
node = nodes[node_params]
if node in already_asked_nodes:
continue
if node["height"] < lowest_height:
continue
already_asked_nodes += [node]
send_to_nm({
"action": "give blocks",
"block_hashes": bytes(b"".join(block_hashes)),
'num': len(block_hashes),
"id": str(uuid4()),
"node": node_params
})
new_message = {
"action": "check blocks download status",
"block_hashes": to_be_downloaded,
"already_asked_nodes": already_asked_nodes,
"id": str(uuid4()),
"time": int(time() + 300)
}
asked = True
put_back_messages.append(new_message)
break
if not asked: #We already asked all applicable nodes
message["time"] = int(time()) + 3600
message["already_asked_nodes"] = []
put_back_messages.append(
message) # we will try to ask again in an hour
if message["action"] == "take nodes list":
for node in message["nodes"]:
if not node in nodes: #Do not overwrite
nodes[node] = {"node": node}
disconnected_nodes = []
for existing_node in nodes:
if not existing_node in message["nodes"]:
disconnected_nodes.append(existing_node)
for dn in disconnected_nodes:
nodes.pop(dn)
if message["action"] == "give nodes list reminder":
_id = str(uuid4())
send_to_nm({
"action": "give nodes list",
"sender": "Blockchain",
"id": _id
})
requests[_id] = "give nodes list"
put_back_messages.append({
"action": "give nodes list reminder",
"time": int(time()) + 3
})
for _message in put_back_messages:
message_queue.put(_message)
try:
check_sync_status(nodes, send_to_nm)
except Exception as e:
logger.error(e)
def wallet(syncer, config):
'''
Wallet is synchronous service which holds private keys and information about
owned outputs. It provides information for transactions and block templates
generation.
'''
def get_height(timeout=2.5):
_id = str(uuid4())
syncer.queues['Notifications'].put({
'action': 'get',
'id': _id,
'key': 'blockchain height',
'sender': "Wallet"
})
message_queue = syncer.queues['Wallet']
start_time = time()
result = None
while True:
put_back = [
] #We wait for specific message, all others will wait for being processed
while not message_queue.empty():
message = message_queue.get()
if (not 'id' in message) or (not message['id'] == _id):
put_back.append(message)
continue
result = message['result']
break
for message in put_back:
message_queue.put(message)
if result:
break
sleep(0.01)
if time() - start_time > timeout:
raise KeyError
if result == 'error':
raise KeyError
return result['value']
notification_cache = {}
def notify(key, value, timestamp=None):
if (key in notification_cache) and (
notification_cache[key]['value'] == value
) and (time() - notification_cache[key]['timestamp']) < 5:
return #Do not spam notifications with the same values
message = {}
message['id'] = uuid4()
message['sender'] = "Wallet"
if not timestamp:
timestamp = time()
message['time'] = timestamp
message['action'] = "set"
message['key'] = key
message['value'] = value
syncer.queues["Notifications"].put(message)
notification_cache[key] = {'value': value, 'timestamp': timestamp}
#set logging
default_log_level = logging.INFO
if "logging" in config: #debug, info, warning, error, critical
loglevels = {
"debug": logging.DEBUG,
"info": logging.INFO,
"warning": logging.WARNING,
"error": logging.ERROR,
"critical": logging.CRITICAL
}
if "base" in config["logging"] and config["logging"][
"base"] in loglevels:
logger.setLevel(loglevels[config["logging"]["base"]])
if "wallet" in config["logging"] and config["logging"][
"wallet"] in loglevels:
#its ok to rewrite
logger.setLevel(loglevels[config["logging"]["wallet"]])
message_queue = syncer.queues['Wallet']
_path = config['location']['wallet']
try:
password = config['wallet'].get('password', None)
except:
password = None
km = KeyDB(path=_path, password=password)
with km.open() as conn:
cursor = conn.cursor()
apply_migrations(cursor)
notify('last wallet update', time())
while True:
sleep(0.01)
while not message_queue.empty():
message = message_queue.get()
if 'action' in message:
logger.info("Process message `%s`" % message['action'])
logger.debug("Process message %s" % message)
else:
logger.info("Process message %s" % message)
if not 'action' in message:
continue
if message['action'] == "process new block":
tx = Transaction(txos_storage=None, excesses_storage=None)
tx.deserialize(
message['tx'], rtx=None, skip_verification=True
) #skip_verification allows us to not provide rtx
block_height = message['height']
last_time_updated = None
with km.open() as conn:
cursor = conn.cursor()
for index in tx.inputs:
#Note it is not check whether output is unspent or not, we check that output is marked as our and unspent in our wallet
if km.is_unspent(index, cursor):
km.spend_output(index, block_height, cursor)
last_time_updated = time()
for _o in tx.outputs:
if km.is_owned_pubkey(_o.address.pubkey.serialize(),
cursor):
km.add_output(_o, block_height, cursor)
last_time_updated = time()
if km.is_saved(_o, cursor):
km.register_processed_output(
_o.serialized_index, block_height, cursor)
if last_time_updated:
notify('last wallet update', last_time_updated)
if message['action'] == "process rollback":
rollback = message['rollback_object']
block_height = message['block_height']
with km.open() as conn:
cursor = conn.cursor()
km.rollback(block_height, cursor)
last_time_updated = time()
notify('last wallet update', last_time_updated)
if message[
'action'] == "process indexed outputs": #during private key import correspondent outputs will be processed again
pass
if message['action'] == "give new taddress":
with km.open() as conn:
cursor = conn.cursor()
address = km.new_address(cursor)
response = {"id": message["id"], "result": address.to_text()}
syncer.queues[message['sender']].put(response)
if message['action'] == "give new address":
with km.open() as conn:
cursor = conn.cursor()
address = km.new_address(cursor)
response = {"id": message["id"], "result": address.serialize()}
syncer.queues[message['sender']].put(response)
if message['action'] == "get confirmed balance stats":
response = {"id": message["id"]}
try:
height = get_height()
with km.open() as conn:
cursor = conn.cursor()
stats = km.get_confirmed_balance_stats(height, cursor)
response["result"] = stats
except KeyError:
response[
"result"] = "error: core_loop didn't set height yet"
except Exception as e:
response["result"] = "error: " + str(e)
syncer.queues[message['sender']].put(response)
if message['action'] == "get confirmed balance list":
response = {"id": message["id"]}
try:
height = get_height()
with km.open() as conn:
cursor = conn.cursor()
stats = km.get_confirmed_balance_list(height, cursor)
response["result"] = stats
except KeyError:
response[
"result"] = "error: core_loop didn't set height yet"
except Exception as e:
response["result"] = "error: " + str(e)
syncer.queues[message['sender']].put(response)
if message['action'] == "give private key":
taddress = message["address"]
a = Address()
a.from_text(taddress)
with km.open() as conn:
cursor = conn.cursor()
priv = km.priv_by_address(a, cursor)
response = {"id": message["id"], "result": priv.private_key}
syncer.queues[message['sender']].put(response)
if message['action'] == "take private key":
response = {"id": message["id"]}
rescan = bool(message["rescan"])
ser_privkey = message["privkey"]
privkey = PrivateKey(ser_privkey, raw=True)
with km.open() as conn:
cursor = conn.cursor()
res = km.add_privkey(privkey, cursor, duplicate_safe=True)
if res and not rescan:
response["result"] = "success"
elif rescan:
response["result"] = "failed"
response["error"] = "rescan is not implemented"
else:
response["result"] = "failed"
syncer.queues[message['sender']].put(response)
continue
if message['action'] == "give last transactions info":
response = {"id": message["id"]}
num = int(message["num"])
with km.open() as conn:
cursor = conn.cursor()
response["result"] = km.give_transactions(num, cursor)
syncer.queues[message['sender']].put(response)
continue
if message['action'] == "generate tx":
response = {"id": message["id"]}
value = int(message["value"])
taddress = message["address"]
a = Address()
a.from_text(taddress)
try:
current_height = get_height()
except KeyError:
response["result"] = "error"
response["error"] = "core_loop didn't set height yet"
syncer.queues[message['sender']].put(response)
continue
except Exception as e:
response["result"] = "error"
response["error"] = str(e)
syncer.queues[message['sender']].put(response)
continue
with km.open() as conn:
cursor = conn.cursor()
_list = km.get_confirmed_balance_list(
current_height, cursor)
list_to_spend = []
summ = 0
utxos = []
expected_fee = 0
for address in _list:
for texted_index in _list[address]:
if summ > value + expected_fee: #TODO fee here
continue
if isinstance(_list[address][texted_index], int):
_index = base64.b64decode(
texted_index.encode())
ser_priv, ser_blinding, apc = km.get_output_private_data(
_index, cursor)
priv = PrivateKey(ser_priv, raw=True)
blinding = PrivateKey(ser_blinding, raw=True)
utxos.append(
(_index, _list[address][texted_index],
priv, blinding, apc))
summ += _list[address][texted_index]
expected_fee += 60000 #Should be enough
if summ < value:
response["result"] = "error"
response["error"] = "Not enough matured coins"
syncer.queues[message['sender']].put(response)
continue
if summ < value + expected_fee:
response["result"] = "error"
response[
"error"] = "Not enough matured coins for value and fees (%.8f)" % (
expected_fee / 1e8)
syncer.queues[message['sender']].put(response)
continue
tx = Transaction(None, None)
tx.add_destination((a, value, True))
tx.blindly_generate(
km.new_address(cursor), utxos,
config["fee_policy"].get("generate_fee_per_kb", 3000))
km.save_generated_transaction(tx, cursor)
response["result"] = tx.serialize()
syncer.queues[message['sender']].put(response)
if message['action'] == "stop":
logger.info("Wallet stops")
return