class SignedStateManager(object):
"""
Signed state manager
"""
def __init__(self, graph):
"""
Create class instance
:param graph: instance of Graph class
:type graph: object
:returns instance of SignedStateManager class
:rtype: object
"""
self.graph = graph
self.crypto = Crypto()
self.transaction = Transaction()
self.logger = logging.getLogger('SignedStateManager')
def get_ordered_state(self, last_round, prev_hash, balance):
""" Gets ordered dict of state
:param last_round: last round of state
:type last_round: int
:param prev_hash: hash of previous state
:type prev_hash: str
:param balance: sorted balance of all nodes
:type balance: sorted dict
:return: ordered state
:rtype: ordered dict
"""
state = collections.OrderedDict([('_id', last_round),
('prev_hash', prev_hash),
('balance', balance)])
return state
def create_state(self, start_round, last_round):
""" Generate state for given round range
:param start_round:
:param last_round:
:return: hash of newly created state
:rtype: str
"""
# Gets Prev_hash
prev_hash = Prisma().db.get_last_state()['hash']
# Gets and sort Balance
balance_dict = Prisma().db.get_account_balance_many(
[start_round, last_round])
balance = collections.OrderedDict(sorted(balance_dict.items()))
state = self.get_ordered_state(last_round, prev_hash, balance)
state_hash = self.crypto.blake_hash(bytes(
dumps(state).encode('utf-8')))
# Result of all old transactions is saved in newly created state, so we can drop tx
Prisma().db.delete_money_transfer_transaction_less_than(last_round)
Prisma().db.insert_state(state, state_hash)
return state_hash
def create_state_sign(self):
"""
Gets unsent state from db or create if it is not exit,
then gets its hash, and finally signs hash and
last round of this state by nodes secret key
:returns: transaction with state signature
:rtype: str
"""
# Get rounds where famousness is fully decided
consensus = Prisma().db.get_consensus_greater_than(
Prisma().db.get_consensus_last_created_sign(),
lim=self.graph.to_sign_count)
self.logger.debug("State consensus %s", str(consensus))
if len(consensus) != self.graph.to_sign_count:
raise ValueError(
"Not enough rounds where famousness is fully decided!")
state_db = Prisma().db.get_state(consensus[-1])
if not state_db:
state_hash = self.create_state(consensus[0], consensus[-1])
else:
# State was generated before
state_hash = state_db['hash']
data = {'last_round': consensus[-1], 'hash': state_hash}
self.logger.debug("State signature data %s", str(data))
sign_data = self.crypto.sign_data(
dumps(data), self.graph.keystore['privateKeySeed'])
# Form transaction
sign_data['type'] = TYPE_SIGNED_STATE
hex_str = self.transaction.hexlify_transaction(sign_data)
Prisma().db.set_consensus_last_created_sign(consensus[-1])
# Save our signature in orrder to send it to new node
data['sign'] = sign_data
Prisma().db.insert_signature(data)
return hex_str
def try_create_state_signatures(self):
"""
While there are enough rounds where famousness is fully decided,
creates state and than digitally sign this
and pushes this signature to list. Afterwards, all created
signatures are inserted into the pool.
:returns: None
"""
state_signatures = []
while self.graph.unsent_count >= self.graph.to_sign_count:
self.logger.debug("Signed state unsent count %s",
str(self.graph.unsent_count))
try:
new_signature = self.create_state_sign()
except ValueError as e:
self.logger.error("Error with get state signature %s", str(e))
break
if new_signature:
state_signatures.append(new_signature)
self.graph.unsent_count -= self.graph.to_sign_count
self.logger.debug("State signature was generated %s",
str(new_signature))
self.logger.debug("Consensus sign response = %s",
str(state_signatures))
self.transaction.insert_transactions_into_pool(state_signatures)
def handle_new_sign(self, tx_dict):
"""
If transaction if valid, stores it
as unchecked to db (that means that we have not
compared this remote hash with our local),
also checks whether there are enough rounds for which
all famous events are completely determined
in order to create the signature of state
On success, sign so many state as we can.
:param tx_dict: parsed transaction dict
:type tx_dict: dict
:returns: None
"""
if tx_dict:
# Validates signature
self.logger.debug("tx_dict: %s", str(tx_dict))
sign_data = self.crypto.validate_state_sign(tx_dict)
del tx_dict['type']
sign_data['sign'] = tx_dict
self.logger.debug("sign_data: %s", str(sign_data))
if (sign_data
and sign_data['last_round'] > self.graph.last_signed_state
and sign_data['sign']['verify_key'] !=
self.graph.keystore['publicKey'].decode('utf-8')):
# Inserts signature as unchecked
Prisma().db.insert_signature_unchecked(sign_data)
# Try to sign local state
while True:
self.logger.debug(
"Handle state signatures last_signed_state %s",
str(self.graph.last_signed_state))
local_consensus = Prisma().db.get_consensus_greater_than(
self.graph.last_signed_state,
lim=self.graph.to_sign_count)
# if there is enough consensus to sign
if (len(local_consensus) != self.graph.to_sign_count
or not self.update_state_sign(local_consensus)):
break
def update_state_sign(self, local_consensus):
"""
Handles signatures of state stored in db
:param local_consensus: local stored rounds where famousness is fully decided
:type local_consensus: tuple
:return: True if signed state is reached and False in opposite case
:rtype: bool
"""
local_signatures = Prisma().db.get_signature(local_consensus[-1])
self.logger.debug("Local signatures %s", str(local_signatures))
if not local_signatures:
self.logger.debug("There is no local signatures")
return False
# Hash of local state
local_hash = Prisma().db.get_state(local_signatures['_id'])['hash']
# Count of successfully checked signatures
unchecked_len = 0
''' Checks if hash of pair is equal to local, on success inserts
signature to db as checked and increments sign count.
Note: pair consists of hash of consensus and its sign. '''
if 'unchecked_pair' in local_signatures:
for pair in local_signatures['unchecked_pair']:
for h, sign in pair.items():
self.logger.debug("Local con hash %s", str(local_hash))
self.logger.debug("Remote hash %s", str(h))
if h == local_hash and ('sign' not in local_signatures
or sign
not in local_signatures['sign']):
self.logger.debug("Consensus hash is equal.")
data = {
'last_round': local_signatures['_id'],
'sign': sign,
'hash': h
}
Prisma().db.insert_signature(data)
unchecked_len += 1
else:
self.logger.error(
"Consensus hash is NOT equal or that signature is already saved."
)
# All stored unchecked signatures are processed now, so we can delete them
Prisma().db.unset_unchecked_signature(local_signatures['_id'])
''' There are no new valid signatures '''
if not unchecked_len:
return False
''' Calculates total count of valid signatures '''
sign_count = unchecked_len
if 'sign' in local_signatures:
sign_count += len(local_signatures['sign'])
''' If there are enough signatures, signs consensus and cleans db '''
if sign_count >= self.graph.min_s:
self.logger.debug("Signs consensus")
Prisma().db.sign_consensus(self.graph.to_sign_count)
self.graph.last_signed_state = local_consensus[-1]
self.logger.debug("self.graph.last_signed_state %s",
str(self.graph.last_signed_state))
# Start cleaning database
self.clean_database(self.graph.last_signed_state)
Prisma().db.set_state_signed(local_signatures['_id'])
return True
else:
return False
def clean_database(self, last_signed):
"""
Deletes signed data from db, so there will never be a huge number of data stored
:param last_signed: last round for which signed state was reached
:type last_signed: int
:return: None
"""
Prisma().db.delete_transaction_less_than(last_signed)
Prisma().db.delete_witnesses_less_than(last_signed)
# Gets list of signed events
hash_list = Prisma().db.get_rounds_hash_list(last_signed)
for _hash in hash_list:
Prisma().db.delete_event(_hash)
Prisma().db.delete_can_see(_hash)
Prisma().db.delete_votes(_hash)
Prisma().db.delete_famous(_hash)
''' We should clear references after removing documents by hash.
In this case we will get much better performance '''
# Delete each link to signed events
Prisma().db.delete_references_can_see(hash_list)
def handle_received_state(self, state, signatures):
""" Validates state received via connection
:param state: state to validate
:type state: dict
:param signatures: list of signatures from more than 1/3 voting power
:type signatures: list
:return: is handling operation successful
"""
last_state_hash = Prisma().db.get_last_state()['hash']
if last_state_hash != state['prev_hash']:
self.logger.error("Recived state have bad hash of prev state")
return False
'''balance = collections.OrderedDict(sorted(state['balance'].items()))
ordered_state = self.get_ordered_state(state['_id'], state['prev_hash'], balance)
self.logger.debug("ordered_state: %s", str(ordered_state))
state_hash = self.crypto.blake_hash(bytes(dumps(ordered_state).encode('utf-8')))'''
state_hash = self.crypto.blake_hash(bytes(
dumps(state).encode('utf-8')))
# TODO improve signature storing and validation
signature_list = []
proof_sign_count = 0
for verify_key in signatures:
# Verifies signed data
temp_dict = {
'verify_key': verify_key,
'signed': signatures[verify_key]
}
sign_data = self.crypto.validate_state_sign(temp_dict)
if not sign_data:
# Incorrect verify_key
return False
node_addr = Prisma().wallet.addr_from_public_key(
bytes(verify_key.encode('utf-8')))
self.logger.debug("node_addr: %s", str(node_addr))
self.logger.debug("node ballance: %s",
str(Prisma().db.get_state_balance(node_addr)))
self.logger.debug(
"sign_data['last_round'] = %s, state['_id'] = %s",
str(sign_data['last_round']), str(state['_id']))
self.logger.debug("sign_data['hash'] = %s, state_hash = %s",
str(sign_data['hash']), str(state_hash))
if (Prisma().db.get_state_balance(node_addr)
and sign_data['last_round'] == state['_id']
and sign_data['hash'] == state_hash):
# All is good add signature to valid list and count it as proof
sign_data['sign'] = temp_dict
signature_list.append(sign_data)
proof_sign_count += 1
else:
self.logger.error("Recived state have bad signature")
# Checks if we get enough signatures
if proof_sign_count >= self.graph.min_s:
Prisma().db.insert_state(state, state_hash, True)
for sign in signature_list:
Prisma().db.insert_signature(sign)
return True
else:
self.logger.error("Recived state have not enough proof signatures")
return False
def handle_received_state_chain(self, chain):
"""
:param chain: Chain of states and signatures as proof
:type chain: list
:return: is handling successful
"""
for stateunit in chain:
if not self.handle_received_state(stateunit['state'],
stateunit['signatures']):
return False
return True
class Graph:
"""
Graph
"""
def __init__(self):
"""
Create class instance
:returns instance of Graph class
:rtype: object
"""
self.wallet = Wallet()
self.keystore = self.wallet.prompt_unlock()
if not self.keystore:
sys.exit(1)
self.logger = logging.getLogger('Graph')
self.logger.debug("Node address %s", str(self.keystore['address']))
self.stake = 1
self.tot_stake = 4 # TODO: It should change dynamically
self.min_s = int(2 * self.tot_stake / 3 + 1) # min stake amount
self.to_sign_count = 10
self.last_signed_state = 0
self.unsent_count = 0
self._cgc = CryptographCommon(graph=self)
self.common = Common()
self.crypto = Crypto()
self.head = None
self.round = {}
self.tbd = set()
self._event = Event(graph=self)
self._fame = Fame(graph=self)
self._order = Order(graph=self)
self._round = Rounds(graph=self)
def init_graph(self):
self.last_signed_state = Prisma().db.get_consensus_last_signed()
self.logger.debug("INIT last_signed_state: %s", str(self.last_signed_state))
is_cg_empty = self.init_events()
self.restore_invariants(is_cg_empty)
if is_cg_empty:
self.sync_genesis()
self.unsent_count = len(Prisma().db.get_consensus_greater_than(
Prisma().db.get_consensus_last_created_sign()))
def init_events(self):
"""
Verifying events stored in database
:return: is cg empty
:rtype: bool
"""
cg = Prisma().db.get_events_many()
if cg:
self.logger.info("Verifying events stored in database.")
for event in cg:
if not self._event.is_valid_event(event, cg[event]):
self.logger.error("Could not verify event with blake2b hash %s",
str(event))
""" Todo: what will we do here if we can not validate an event in database? """
exit()
return False
else:
return True
def restore_invariants(self, is_cg_empty):
"""
Initializes cryptograph if it is empty
:param is_cg_empty: is cryptograph empty
(this arg func get from init_events)
:return: None
"""
if is_cg_empty:
h, ev = self._event.new_event([], ())
self._event.add_event(h, ev)
Prisma().db.insert_round({h: 0})
Prisma().db.insert_witness({0: {ev.c: h}})
Prisma().db.insert_can_see({h: {ev.c: h}})
Prisma().db.insert_head(h)
else:
self.logger.debug("Reconnect")
def sync_genesis(self):
"""
Insert genesis state if it is not exist
"""
if not Prisma().db.get_state(-1):
gen_state = Common().read_genesis_state()
self.logger.debug("genesis_state: %s", str(gen_state))
Prisma().db.insert_state(gen_state['state'], gen_state['hash'], gen_state['signed'])
def signed_event_response(self):
"""
Sends the events we know about without actually sending the events.
As a response we shall get the events in full which we don't know about.
The remote peer will call local_cryptograph_response() and create a subset based
on this information. signed_events contains two parents and their heights.
:returns: signed events
:rtype: dict
"""
head = Prisma().db.get_head()
if head:
signed_events = self.crypto.sign_data(
json.dumps({c: Prisma().db.get_height(h) for c, h in Prisma().db.get_can_see(head).items()}),
self.keystore['privateKeySeed'])
if signed_events:
return signed_events
return None
def get_clean_remote_cg(self, remote_cg):
"""
Removes events from remote cg we have already signed or know.
:param: remote cryptograph with events that we might know
:type remote_cg: tuple
:returns: remote cg without events that we have already sign or know
:rtype: dict
"""
remote_cg = self._event.restore(remote_cg)
for event_hash in list(remote_cg):
event_round = Prisma().db.get_round(event_hash)
if event_round == -1:
self.logger.error("Could not clean remote cg: get event round ERROR !")
return False
if (event_round and event_round <= self.last_signed_state) \
or Prisma().db.get_event(event_hash, as_tuple=False):
self.logger.debug("CLEANING DELETE hash = %s", str(event_hash))
del remote_cg[event_hash]
self.logger.debug("Clean remote HG %s", str(remote_cg))
return remote_cg
def validate_add_event(self, events_sign):
"""
Gets remote events signature, and validate than delete events that we already know
:param events_sign: signature of remote cryptograph
:type events_sign: dict
:returns: remote cryptograph without events that we already know and remote head
:rtype: dict and str
"""
msg = self.crypto.verify_concatenated(events_sign)
remote_head, remote_cg = json.loads(msg.decode('utf-8'))
self.logger.debug("VALIDATE remote_cg %s", str(remote_cg))
self.logger.debug("VALIDATE remote_head %s", str(remote_head))
remote_cg = self.get_clean_remote_cg(remote_cg)
return remote_cg, remote_head
def insert_new_events(self, remote_cg, remote_head, payload):
"""
Inserts new remote events into db and create new event of that sync
:param remote_cg: remote cryptograph
:type remote_cg: dict
:param remote_head: hash of head of remote cryptograph
:type remote_head: str
:param payload: data (transaction) that will be inserted into the new event
:type payload: dict or tuple
:returns: topologicaly sorted sequence of new events to process.
:rtype: events: set
"""
new = tuple(self._cgc.toposort(remote_cg.keys(), lambda u: remote_cg[u].p))
self.logger.debug("remote_cg %s", str(remote_cg))
self.logger.debug("remote_head %s", str(remote_head))
self.logger.debug("payload %s", str(payload))
self.logger.debug("new %s", str(new))
for h in new:
ev = remote_cg[h]
self.logger.debug("h %s", str(h))
self.logger.debug("ev %s", str(ev))
if self._event.is_valid_event(h, ev):
self._event.add_event(h, ev)
else:
self.logger.debug("Event not valid: %s", str(ev))
if self._event.is_valid_event(remote_head, remote_cg[remote_head]):
h, ev = self._event.new_event(payload, (Prisma().db.get_head(), remote_head))
self.logger.debug("new_event_event_part %s", str(ev))
assert self._event.is_valid_event(h, ev)
if self._event.add_event(h, ev):
Prisma().db.insert_head(h)
return new + (h,)
return False
def local_cryptograph_response(self, signed_event_response):
"""
Based on the get_event_response and the data generated in signed_event_response()
on the remote peer we calculate a subset of events that the asking node does not
know about. The two parents and their height are used below. Returns a dict;
see crypto.py
:param signed_event_response: signed remote events and last remote event time
:type signed_event_response: dict
:returns: signed local events or False if error
:rtype: dict or bool
"""
head = Prisma().db.get_head()
self.logger.debug("signed_event_response %s", str(signed_event_response))
self.logger.debug("head %s", str(head))
# cg is a list of event tuples, it should be a dict of tuples
if head:
cs = json.loads((self.crypto.verify_concatenated(signed_event_response)).decode('utf-8'))
# cs are a dict
subset = {h: Prisma().db.get_event(h)
for h in self._cgc.bfs((head,),
lambda u: (p for p in
Prisma().db.get_event(u, clear_parent=True).p
if Prisma().db.get_event(p).c not in cs or
Prisma().db.get_height(p) > cs[Prisma().db.get_event(p).c]))}
response = json.dumps((head, subset))
local_cryptograph_response_res = self.crypto.sign_data(response, self.keystore['privateKeySeed'])
self.logger.debug("local_cryptograph_response_res %s", str(local_cryptograph_response_res))
return local_cryptograph_response_res
return False
class Event(object):
"""
Event
"""
def __init__(self, graph):
"""
Create class instance
:param graph: instance of cryptograph class
:type graph: object
:returns instance of Event class
:rtype: object
"""
self.graph = graph
self.crypto = Crypto()
self.logger = logging.getLogger('Event')
@staticmethod
def restore(msg):
"""
Restore Named tuple Event from tuple
:param msg: event dict formatted as tuple(not named) with order that match with order in named Event
:type msg: tuple
:return: formated Event
:rtype: named tuple(Event)
"""
res = {}
for h in msg.keys():
list_event = msg[h]
res[h] = Event_(list_event[0], tuple(list_event[1]), list_event[2],
list_event[3], list_event[4])
return res
def new_event(self, d, p):
"""
Create a new event (and also return it's hash)
Events are vertices in the hash graph containing
the info(hashes) about 2 parent graphs that synced/gossiped
:param d: data/payload
:type d: tuple
:param p: event-hash of 2 parents
:type p: set
:return: hash of new event and event itself
:rtype: str and named tuple
"""
self.logger.debug("NEW_EVENT: %s %s", str(d), str(p))
t = time()
s = self.crypto.sign_data(
dumps(d, p, t, self.graph.keystore['publicKey'].decode()),
self.graph.keystore['privateKeySeed'], True)
self.logger.debug("Sign: %s", s['sig_detached'])
ev = Event_(d, p, t, s['verify_key'], s['sig_detached'])
self.logger.debug("Created event : %s", str(ev))
return self.crypto.blake_hash(bytes(dumps(ev).encode('utf-8'))), ev
def is_valid_event(self, blake2hash, ev):
"""
Validate event
:param blake2hash: event hash
:type blake2hash: str
:param ev: Event = named tuple containing : d p c t s
:type ev: named tuple
:return: True if event valid and False otherwise
:rtype: bool
"""
if not self.crypto.verify_local_event(ev):
return False
self.logger.debug("VALID_CHECK %s %s", str(blake2hash), str(ev))
self.logger.debug(
"self.hash %s",
str(self.crypto.blake_hash(bytes(dumps(ev).encode('utf-8')))))
if self.crypto.blake_hash(bytes(
dumps(ev).encode('utf-8'))) == blake2hash:
self.logger.debug("Event blake2b hash matches ")
else:
self.logger.debug("HASHES dont match.")
self.logger.debug("Parents: %s", str(ev.p))
if ev.p != ():
rnd1 = Prisma().db.get_round(ev.p[0])
rnd2 = Prisma().db.get_round(ev.p[1])
first_parent = Prisma().db.get_event(ev.p[0], as_tuple=False)
second_parent = Prisma().db.get_event(ev.p[1], as_tuple=False)
if (self.crypto.blake_hash(bytes(
dumps(ev).encode('utf-8'))) == blake2hash
and (ev.p == () or
(len(ev.p) == 2 and
((first_parent and first_parent[ev.p[0]]['c'] == ev.c)
or rnd1 <= self.graph.last_signed_state) and
((second_parent and second_parent[ev.p[1]]['c'] != ev.c)
or rnd2 <= self.graph.last_signed_state)))):
self.logger.debug("Event successfully validated: %s", str(ev))
return True
self.logger.debug("Event could not be successfully validated: %s",
str(ev))
return False
# TODO: check if there is a fork (rly need reverse edges?)
# and all(self.cg[x].c != ev.c
# for x in self.preds[ev.p[0]]))))
def add_event(self, blake2hash, ev):
"""
Save event to database
:param blake2hash: event hash
:type blake2hash: str
:param ev: Event = named tuple containing : d p c t s
:type ev: named tuple
:return: True if successfully added and False otherwise
:rtype: bool
"""
try:
Prisma().graph.tbd.add(blake2hash)
if ev.p == ():
if not Prisma().db.insert_height({blake2hash: 0}):
self.logger.error(
"Could not add root event with blake2b hash %s.",
str(blake2hash))
return False
else:
height_list = []
for p in ev.p:
height_list.append(Prisma().db.get_height(p))
if not Prisma().db.insert_height(
{blake2hash: max(height_list) + 1}):
self.logger.debug(
"Could not add new event with blake2b hash %s",
str(blake2hash))
return False
Prisma().db.insert_event({blake2hash: ev})
except Exception as e:
self.logger.error("Could not add new event. Reason:", e)
return False
return True