def __init__(self,
network=None,
domain_id=None,
workingdir=".",
loglevel="all",
logname=None):
if network is not None:
self.network = network
self.core = network.core
self.stats = network.core.stats
self.data_handler = network.domains[domain_id]['data']
else:
self.stats = BBcStats()
self.repair_log = os.path.join(workingdir, domain_id.hex(),
"repair_log.json")
self.logger = logger.get_logger(key="repair_manager",
level=loglevel,
logname=logname)
self.domain_id = domain_id
self.queue = queue.Queue()
self.requesting_list = dict()
self.loop_flag = True
th_nw_loop = threading.Thread(target=self._manager_loop)
th_nw_loop.setDaemon(True)
th_nw_loop.start()
class RepairManager:
"""Data repair manager for forged transaction/asset"""
REQUEST_REPAIR_TRANSACTION = 0
REQUEST_REPAIR_ASSET_FILE = 1
REQUEST_TO_SEND_TRANSACTION_DATA = 2
RESPONSE_TRANSACTION_DATA = 3
REQUEST_TO_SEND_ASSET_FILE = 4
RESPONSE_ASSET_FILE = 5
def __init__(self,
network=None,
domain_id=None,
workingdir=".",
loglevel="all",
logname=None):
if network is not None:
self.network = network
self.core = network.core
self.stats = network.core.stats
self.data_handler = network.domains[domain_id]['data']
else:
self.stats = BBcStats()
self.repair_log = os.path.join(workingdir, domain_id.hex(),
"repair_log.json")
self.logger = logger.get_logger(key="repair_manager",
level=loglevel,
logname=logname)
self.domain_id = domain_id
self.queue = queue.Queue()
self.requesting_list = dict()
self.loop_flag = True
th_nw_loop = threading.Thread(target=self._manager_loop)
th_nw_loop.setDaemon(True)
th_nw_loop.start()
def _output_log(self, repair_info):
"""Output log in json format"""
with open(self.repair_log, "a") as f:
f.write(json.dumps(repair_info) + "\n")
def exit_loop(self):
"""Exit the manager loop"""
self.loop_flag = False
self.put_message()
def _manager_loop(self):
"""Main loop"""
while self.loop_flag:
msg = self.queue.get()
if msg is None:
continue
if msg[KeyType.
command] == RepairManager.REQUEST_REPAIR_TRANSACTION:
self._repair_transaction_data(msg[KeyType.transaction_id])
elif msg[KeyType.
command] == RepairManager.REQUEST_REPAIR_ASSET_FILE:
self._repair_asset_file(msg[KeyType.asset_group_id],
msg[KeyType.asset_id])
elif msg[
KeyType.
command] == RepairManager.REQUEST_TO_SEND_TRANSACTION_DATA:
self._send_transaction_data(msg)
elif msg[KeyType.
command] == RepairManager.RESPONSE_TRANSACTION_DATA:
self._receive_transaction_data_from_others(msg)
elif msg[KeyType.
command] == RepairManager.REQUEST_TO_SEND_ASSET_FILE:
self._send_asset_file(msg)
elif msg[KeyType.command] == RepairManager.RESPONSE_ASSET_FILE:
self._receive_asset_file_from_others(msg)
def put_message(self, msg=None):
"""append a message to the queue"""
self.queue.put(msg)
def _repair_transaction_data(self, transaction_id):
"""Repair forged transaction_data or asset_file by getting legitimate one from other nodes
Args:
transaction_id (bytes): target transaction_id
"""
#print("_repair_transaction_data:")
self.stats.update_stats_increment("transaction", "repair_request", 1)
forged_asset_files = set()
if len(self.data_handler.db_adaptors) > 1:
valid_txobj = None
db_nums_with_invalid_data = list()
for idx in range(1, len(self.data_handler.db_adaptors)):
result_txobj, result_asset_files = self.data_handler.search_transaction(
transaction_id=transaction_id, db_num=idx)
txobj_is_valid, valid_assets, invalid_assets = bbclib.validate_transaction_object(
result_txobj[0], result_asset_files)
if txobj_is_valid and valid_txobj is None:
valid_txobj = result_txobj[0]
if not txobj_is_valid:
db_nums_with_invalid_data.append(idx)
if len(invalid_assets) > 0:
for ent in invalid_assets:
forged_asset_files.add(ent)
if valid_txobj is None:
self.stats.update_stats_increment("transaction",
"fail_to_repair_in_local", 1)
self.logger.fatal(
"Failed to repair transaction locally (transaction_id=%s in domain=%s)"
% (transaction_id.hex(), self.domain_id.hex()))
else:
for i in db_nums_with_invalid_data:
self.data_handler.restore_transaction_data(
db_num=i,
transaction_id=transaction_id,
txobj=valid_txobj)
self.stats.update_stats_increment("transaction",
"success_repair", 1)
self._output_log({
"transaction_id": transaction_id.hex(),
"request_at": int(time.time()),
"repaired_by": "locally",
"repaired_at": int(time.time())
})
if len(forged_asset_files) > 0:
for asgid, ast in forged_asset_files:
self._repair_asset_file(asset_group_id=asgid,
asset_id=ast,
need_check=False)
if self.data_handler.replication_strategy == DataHandler.REPLICATION_EXT:
return
random_nonce = bbclib.get_random_value(4)
while random_nonce in self.requesting_list:
random_nonce = bbclib.get_random_value(4)
self.requesting_list[random_nonce] = {
"transaction_id": transaction_id.hex(),
"request_at": int(time.time())
}
msg = {
KeyType.domain_id: self.domain_id,
KeyType.infra_msg_type: InfraMessageCategory.CATEGORY_DATA,
KeyType.infra_command: DataHandler.REPAIR_TRANSACTION_DATA,
KeyType.command: RepairManager.REQUEST_TO_SEND_TRANSACTION_DATA,
KeyType.transaction_id: transaction_id,
KeyType.nonce: random_nonce,
}
self.network.broadcast_message_in_network(
domain_id=self.domain_id,
payload_type=PayloadType.Type_any,
msg=msg)
return
def _repair_asset_file(self, asset_group_id, asset_id, need_check=True):
"""Repair forged asset_file by getting legitimate one from other nodes
Args:
asset_group_id (bytes): asset_group_id of the asset
asset_id (bytes): asset_id of the asset
need_check (bool): If True, check the digest of the asset file
"""
#print("_repair_asset_file:")
if self.data_handler.use_external_storage:
return
if need_check:
asset_file = self.data_handler.get_in_storage(
asset_group_id, asset_id)
if asset_file is not None and asset_id == hashlib.sha256(
asset_file).digest():
return
random_nonce = bbclib.get_random_value(4)
while random_nonce in self.requesting_list:
random_nonce = bbclib.get_random_value(4)
self.requesting_list[random_nonce] = {
"asset_group_id": asset_group_id.hex(),
"asset_id": asset_id.hex(),
"request_at": int(time.time())
}
msg = {
KeyType.domain_id: self.domain_id,
KeyType.infra_msg_type: InfraMessageCategory.CATEGORY_DATA,
KeyType.infra_command: DataHandler.REPAIR_TRANSACTION_DATA,
KeyType.command: RepairManager.REQUEST_TO_SEND_ASSET_FILE,
KeyType.asset_group_id: asset_group_id,
KeyType.asset_id: asset_id,
KeyType.nonce: random_nonce,
}
self.network.broadcast_message_in_network(
domain_id=self.domain_id,
payload_type=PayloadType.Type_any,
msg=msg)
def _send_transaction_data(self, dat):
"""Send transaction data if having valid one"""
#print("_send_transaction_data::")
transaction_id = dat[KeyType.transaction_id]
for idx in range(len(self.data_handler.db_adaptors)):
result_txobj, result_asset_files = self.data_handler.search_transaction(
transaction_id=transaction_id, db_num=idx)
txobj_is_valid, valid_assets, invalid_assets = bbclib.validate_transaction_object(
result_txobj[transaction_id])
if txobj_is_valid:
dat[KeyType.command] = RepairManager.RESPONSE_TRANSACTION_DATA
dat[KeyType.transaction_data] = bbclib.serialize(
result_txobj[transaction_id])
dat[KeyType.destination_node_id] = dat[KeyType.source_node_id]
self.network.send_message_in_network(None,
domain_id=self.domain_id,
msg=dat)
return
def _receive_transaction_data_from_others(self, dat):
"""Receive transaction data from other core_nodes and check its validity
Args:
dat (dict): received message
"""
#print("_receive_transaction_data_from_others:")
if KeyType.transaction_data not in dat or KeyType.transaction_id not in dat or KeyType.nonce not in dat:
return
if dat[KeyType.nonce] not in self.requesting_list:
return
asset_files = dict()
if KeyType.all_asset_files in dat:
asset_files = dat[KeyType.all_asset_files]
txobj, fmt_type = bbclib.deserialize(dat[KeyType.transaction_data])
if txobj.transaction_data is None:
return
txobj_is_valid, valid_assets, invalid_assets = bbclib.validate_transaction_object(
txobj, asset_files)
if txobj_is_valid:
self.stats.update_stats_increment("transaction", "success_repair",
1)
for idx in range(len(self.data_handler.db_adaptors)):
self.data_handler.restore_transaction_data(
db_num=idx,
transaction_id=txobj.transaction_id,
txobj=txobj)
add_info = {
"repaired_by": dat[KeyType.source_node_id].hex(),
"repaired_at": int(time.time())
}
self.requesting_list[dat[KeyType.nonce]].update(add_info)
self._output_log(self.requesting_list[dat[KeyType.nonce]])
del self.requesting_list[dat[KeyType.nonce]]
def _send_asset_file(self, dat):
"""Send the asset file if having valid one
Args:
dat (dict): received message
"""
#print("_send_asset_file::")
asset_group_id = dat[KeyType.asset_group_id]
asset_id = dat[KeyType.asset_id]
asset_file = self.data_handler.get_in_storage(asset_group_id, asset_id)
if asset_file is None:
return
result_txobj, result_asset_files = self.data_handler.search_transaction(
asset_group_id=asset_group_id, asset_id=asset_id)
txobj = next(iter(result_txobj.values()))
txobj_is_valid, valid_assets, invalid_assets = bbclib.validate_transaction_object(
txobj, result_asset_files)
if (asset_group_id, asset_id) in valid_assets:
dat[KeyType.command] = RepairManager.RESPONSE_ASSET_FILE
dat[KeyType.asset_group_id] = asset_group_id
dat[KeyType.asset_id] = asset_id
dat[KeyType.asset_file] = asset_file
dat[KeyType.destination_node_id] = dat[KeyType.source_node_id]
self.network.send_message_in_network(None,
domain_id=self.domain_id,
msg=dat)
def _receive_asset_file_from_others(self, dat):
"""Receive asset file from other core_nodes and check its validity
Args:
dat (dict): received message
"""
#print("_receive_asset_file_from_others:")
if KeyType.nonce not in dat or dat[
KeyType.nonce] not in self.requesting_list:
return
if KeyType.asset_group_id not in dat or KeyType.asset_id not in dat or KeyType.asset_file not in dat:
return
asset_group_id = dat[KeyType.asset_group_id]
asset_id = dat[KeyType.asset_id]
asset_file = dat[KeyType.asset_file]
if asset_file is None:
return
asset_files = {asset_id: asset_file}
result_txobj, result_asset_files = self.data_handler.search_transaction(
asset_group_id=asset_group_id, asset_id=asset_id)
txobj = next(iter(result_txobj.values()))
txobj_is_valid, valid_assets, invalid_assets = bbclib.validate_transaction_object(
txobj, asset_files)
if (asset_group_id, asset_id) in valid_assets:
self.data_handler.store_in_storage(asset_group_id,
asset_id,
asset_file,
do_overwrite=True)
add_info = {
"repaired_by": dat[KeyType.source_node_id].hex(),
"repaired_at": int(time.time())
}
self.requesting_list[dat[KeyType.nonce]].update(add_info)
self._output_log(self.requesting_list[dat[KeyType.nonce]])
del self.requesting_list[dat[KeyType.nonce]]
class RepairManager:
"""
Data repair messager for forged transaction/asset
"""
REQUEST_REPAIR_TRANSACTION = 0
REQUEST_TO_SEND_TRANSACTION_DATA = 1
RESPONSE_TRANSACTION_DATA = 2
def __init__(self, network=None, domain_id=None, workingdir=".", loglevel="all", logname=None):
if network is not None:
self.network = network
self.core = network.core
self.stats = network.core.stats
self.data_handler = network.domains[domain_id]['data']
else:
self.stats = BBcStats()
self.repair_log = os.path.join(workingdir, domain_id.hex(), "repair_log.json")
self.logger = logger.get_logger(key="repair_manager", level=loglevel, logname=logname)
self.domain_id = domain_id
self.queue = queue.Queue()
self.requesting_list = dict()
self.loop_flag = True
th_nw_loop = threading.Thread(target=self.manager_loop)
th_nw_loop.setDaemon(True)
th_nw_loop.start()
def output_log(self, repair_info):
with open(self.repair_log, "a") as f:
f.write(json.dumps(repair_info)+"\n")
def exit_loop(self):
self.loop_flag = False
self.put_message()
def manager_loop(self):
while self.loop_flag:
msg = self.queue.get()
if msg is None:
continue
if msg[KeyType.command] == RepairManager.REQUEST_REPAIR_TRANSACTION:
self.repair_transaction_data(msg[KeyType.transaction_id])
elif msg[KeyType.command] == RepairManager.REQUEST_TO_SEND_TRANSACTION_DATA:
self.send_transaction_data(msg)
elif msg[KeyType.command] == RepairManager.RESPONSE_TRANSACTION_DATA:
self.receive_transaction_data_from_others(msg)
def put_message(self, msg=None):
self.queue.put(msg)
def repair_transaction_data(self, transaction_id):
"""
Repair forged transaction_data or asset_file by getting legitimate one from other nodes
:param domain_id:
:param transaction_id:
:return:
"""
#print("repair_transaction_data:")
self.stats.update_stats_increment("transaction", "repair_request", 1)
if len(self.data_handler.db_adaptors) > 1:
valid_txobj = None
valid_asset_files = dict()
db_nums_with_invalid_data = list()
for idx in range(1, len(self.data_handler.db_adaptors)):
result_txobj, result_asset_files = self.data_handler.search_transaction(transaction_id=transaction_id, db_num=idx)
txobj_is_valid, valid_assets, invalid_assets = bbclib.validate_transaction_object(result_txobj[0],
result_asset_files)
if txobj_is_valid and valid_txobj is None:
valid_txobj = result_txobj[0]
if not txobj_is_valid:
db_nums_with_invalid_data.append(idx)
for ast, file in valid_assets.items():
valid_asset_files[ast] = file
if valid_txobj is None:
self.stats.update_stats_increment("transaction", "fail_to_repair_in_local", 1)
self.logger.fatal("Failed to repair transaction locally (transaction_id=%s in domain=%s)" % (transaction_id.hex(),
self.domain_id.hex()))
else:
for i in db_nums_with_invalid_data:
self.data_handler.restore_data(db_num=i, transaction_id=transaction_id, txobj=valid_txobj, asset_files=valid_assets)
self.stats.update_stats_increment("transaction", "success_repair", 1)
self.output_log({"transaction_id": transaction_id.hex(), "request_at": int(time.time()),
"repaired_by": "locally", "repaired_at": int(time.time())})
if self.data_handler.replication_strategy == DataHandler.REPLICATION_EXT:
return
random_nonce = bbclib.get_random_value(4)
while random_nonce in self.requesting_list:
random_nonce = bbclib.get_random_value(4)
self.requesting_list[random_nonce] = {
"transaction_id": transaction_id.hex(),
"request_at": int(time.time())
}
msg = {
KeyType.domain_id: self.domain_id,
KeyType.infra_msg_type: InfraMessageCategory.CATEGORY_DATA,
KeyType.infra_command: DataHandler.REPAIR_TRANSACTION_DATA,
KeyType.command: RepairManager.REQUEST_TO_SEND_TRANSACTION_DATA,
KeyType.transaction_id: transaction_id,
KeyType.nonce: random_nonce,
}
self.network.broadcast_message_in_network(domain_id=self.domain_id,
payload_type=PayloadType.Type_any, msg=msg)
return
def send_transaction_data(self, dat):
"""
Send transaction data if having valid one
:param dat:
:return:
"""
#print("send_transaction_data::")
transaction_id = dat[KeyType.transaction_id]
for idx in range(len(self.data_handler.db_adaptors)):
result_txobj, result_asset_files = self.data_handler.search_transaction(transaction_id=transaction_id, db_num=idx)
txobj_is_valid, valid_assets, invalid_assets = bbclib.validate_transaction_object(result_txobj[transaction_id],
result_asset_files)
if txobj_is_valid and len(invalid_assets) == 0:
dat[KeyType.command] = RepairManager.RESPONSE_TRANSACTION_DATA
dat[KeyType.transaction_data] = result_txobj[transaction_id].transaction_data
if len(valid_assets) > 0:
dat[KeyType.all_asset_files] = valid_assets
else:
if KeyType.all_asset_files in dat:
del dat[KeyType.all_asset_files]
dat[KeyType.destination_node_id] = dat[KeyType.source_node_id]
self.network.send_message_in_network(None, domain_id=self.domain_id, msg=dat)
return
def receive_transaction_data_from_others(self, dat):
"""
Receive transaction data from other core_nodes and check its validity
:param dat:
:return:
"""
#print("receive_transaction_data_from_others:")
if KeyType.transaction_data not in dat or KeyType.transaction_id not in dat or KeyType.nonce not in dat:
return
if dat[KeyType.nonce] not in self.requesting_list:
return
asset_files = dict()
if KeyType.all_asset_files in dat:
asset_files = dat[KeyType.all_asset_files]
txobj = bbclib.BBcTransaction(deserialize=dat[KeyType.transaction_data])
if txobj.transaction_data is None:
return
txobj_is_valid, valid_assets, invalid_assets = bbclib.validate_transaction_object(txobj, asset_files)
if txobj_is_valid and len(invalid_assets) == 0:
self.stats.update_stats_increment("transaction", "success_repair", 1)
for idx in range(len(self.data_handler.db_adaptors)):
self.data_handler.restore_data(db_num=idx, transaction_id=txobj.transaction_id,
txobj=txobj, asset_files=valid_assets)
add_info = {
"repaired_by": dat[KeyType.source_node_id].hex(),
"repaired_at": int(time.time())
}
self.requesting_list[dat[KeyType.nonce]].update(add_info)
self.output_log(self.requesting_list[dat[KeyType.nonce]])
del self.requesting_list[dat[KeyType.nonce]]