def init_lg_repro_drop_data(drop: dict):
"""
Creates and appends per-drop reproducibility information at the logical graph stage.
:param drop:
:return: The same drop with appended reproducibility information
"""
level = rflag_caster(drop["reprodata"]["rmode"])
if not rmode_supported(level):
logger.warning("Requested reproducibility mode %s not yet implemented",
str(level))
level = REPRO_DEFAULT
drop["reprodata"]["rmode"] = str(level.value)
if level == ReproducibilityFlags.ALL:
for rmode in ALL_RMODES:
data = accumulate_lg_drop_data(drop, rmode)
merkletree = MerkleTree(data.items(), common_hash)
data["merkleroot"] = merkletree.merkle_root
drop["reprodata"][rmode.name]["lg_data"] = data
drop["reprodata"][rmode.name]["lg_parenthashes"] = {}
else:
data = accumulate_lg_drop_data(drop, level)
merkletree = MerkleTree(data.items(), common_hash)
data["merkleroot"] = merkletree.merkle_root
drop["reprodata"]["lg_data"] = data
drop["reprodata"]["lg_parenthashes"] = {}
return drop
def build_pgt_block_data(drop: dict, rmode=None):
"""
Builds the physical graph template reprodata entry for a processed drop description
:param drop: The drop description
:return:
"""
if rmode is None:
block_data = []
if "pgt_data" in drop["reprodata"]:
if "merkleroot" in drop["reprodata"]["pgt_data"]:
block_data.append(drop["reprodata"]["pgt_data"]["merkleroot"])
if "lg_blockhash" in drop["reprodata"]:
block_data.append(drop["reprodata"]["lg_blockhash"])
for parenthash in sorted(
drop["reprodata"]["pgt_parenthashes"].values()):
block_data.append(parenthash)
mtree = MerkleTree(block_data, common_hash)
drop["reprodata"]["pgt_blockhash"] = mtree.merkle_root
else:
block_data = []
if "pgt_data" in drop["reprodata"][rmode.name]:
if "merkleroot" in drop["reprodata"][rmode.name]["pgt_data"]:
block_data.append(
drop["reprodata"][rmode.name]["pgt_data"]["merkleroot"])
if "lg_blockhash" in drop["reprodata"][rmode.name]:
block_data.append(drop["reprodata"][rmode.name]["lg_blockhash"])
for parenthash in sorted(
drop["reprodata"][rmode.name]["pgt_parenthashes"].values()):
block_data.append(parenthash)
mtree = MerkleTree(block_data, common_hash)
drop["reprodata"][rmode.name]["pgt_blockhash"] = mtree.merkle_root
def __init__(self,
blockchain=None,
name: str = 'block',
height: int = 0,
version: int = 0,
coinbase: Corner = None,
corners: list = None,
timestamp=0,
previous_hash=pycryptonight.cn_slow_hash(b''),
nonce=(0).to_bytes(NONCE_SIZE, 'big')):
"""
Initialises a Block instance.
:param blockchain: Blockchain
:param name: str
:param height: int
:param version: int
:param coinbase: Corner
:param corners: Corner list
:param timestamp: int
:param previous_hash: bytes
:param nonce: bytes
"""
super().__init__(f"{name} - {height} :")
self.blockchain = blockchain
self.version = version
self.timestamp = timestamp
self.previous_hash = previous_hash
self.nonce = nonce
self.coinbase = coinbase
self._corners = [] if corners is None else corners
self.merkle_tree = MerkleTree(self.corners, fast_hash)
self.hash = self.get_hash()
self.difficulty = 4
def build_rg_block_data(drop: dict, rmode=None):
"""
Builds the runtime graph reprodata entry for a processed drop description.
:param drop: The drop description
:return:
"""
if rmode is None:
block_data = [
drop["reprodata"]["rg_data"]["merkleroot"],
drop["reprodata"]["pg_blockhash"],
drop["reprodata"]["pgt_blockhash"],
drop["reprodata"]["lg_blockhash"],
]
for parenthash in sorted(
drop["reprodata"]["rg_parenthashes"].values()):
block_data.append(parenthash)
mtree = MerkleTree(block_data, common_hash)
drop["reprodata"]["rg_blockhash"] = mtree.merkle_root
else:
import json
block_data = [
drop["reprodata"][rmode.name].get(
"rg_data", {"merkleroot": b""})["merkleroot"],
drop["reprodata"][rmode.name]["pg_blockhash"],
drop["reprodata"][rmode.name]["pgt_blockhash"],
drop["reprodata"][rmode.name]["lg_blockhash"],
]
for parenthash in sorted(
drop["reprodata"][rmode.name]["rg_parenthashes"].values()):
block_data.append(parenthash)
mtree = MerkleTree(block_data, common_hash)
drop["reprodata"][rmode.name]["rg_blockhash"] = mtree.merkle_root
def system_summary():
"""
Summarises the system this function is run on.
Includes system, cpu, gpu and module details
:return: A dictionary of system details
"""
merkletree = MerkleTree()
system_info = {}
uname = platform.uname()
system_info["system"] = {
"system": uname.system,
"release": uname.release,
"machine": uname.machine,
"processor": uname.processor,
}
cpu_freq = psutil.cpu_freq()
system_info["cpu"] = {
"cores_phys": psutil.cpu_count(logical=False),
"cores_logic": psutil.cpu_count(logical=True),
"max_frequency": cpu_freq.max,
"min_frequency": cpu_freq.min,
}
sys_mem = psutil.virtual_memory()
system_info["memory"] = {"total": sys_mem.total}
gpus = GPUtil.getGPUs()
system_info["gpu"] = {}
for gpu in gpus:
system_info["gpu"][gpu.id] = {
"name": gpu.name,
"memory": gpu.memoryTotal
}
system_info["modules"] = find_loaded_modules()
merkletree.append(list(system_info.items()))
system_info["signature"] = merkletree.merkle_root
return system_info
def system_summary():
"""
Summarises the system this function is run on.
Includes system, cpu, gpu and module details
:return: A dictionary of system details
"""
merkletree = MerkleTree()
system_info = {}
uname = platform.uname()
system_info['system'] = {
'system': uname.system,
'release': uname.release,
'machine': uname.machine,
'processor': uname.processor
}
cpu_freq = psutil.cpu_freq()
system_info['cpu'] = {
'cores_phys': psutil.cpu_count(logical=False),
'cores_logic': psutil.cpu_count(logical=True),
'max_frequency': cpu_freq.max,
'min_frequency': cpu_freq.min
}
sys_mem = psutil.virtual_memory()
system_info['memory'] = {'total': sys_mem.total}
gpus = GPUtil.getGPUs()
system_info['gpu'] = {}
for gpu in gpus:
system_info['gpu'][gpu.id] = {
'name': gpu.name,
'memory': gpu.memoryTotal
}
system_info['modules'] = find_loaded_modules()
merkletree.append([system_info[item] for item in system_info])
system_info['signature'] = merkletree.merkle_root
return system_info
def init_lgt_repro_drop_data(drop: dict, level: ReproducibilityFlags):
"""
Creates and appends per-drop reproducibility information at the logical template stage.
:param drop:
:param level:
:return: The same drop with appended reproducibility information.
"""
# Catch pre-set per-drop rmode
if "reprodata" in drop.keys():
if "rmode" in drop["reprodata"].keys():
level = rflag_caster(drop["reprodata"]["rmode"])
else:
drop["reprodata"] = {"rmode": str(level.value)}
if level == ReproducibilityFlags.ALL:
for rmode in ALL_RMODES:
data = accumulate_lgt_drop_data(drop, rmode)
merkletree = MerkleTree(data.items(), common_hash)
data["merkleroot"] = merkletree.merkle_root
drop["reprodata"][rmode.name] = {
"rmode": str(rmode.value),
"lgt_data": data,
"lg_parenthashes": {},
}
else:
data = accumulate_lgt_drop_data(drop, level)
merkletree = MerkleTree(data.items(), common_hash)
data["merkleroot"] = merkletree.merkle_root
drop["reprodata"] = {
"rmode": str(level.value),
"lgt_data": data,
"lg_parenthashes": {},
}
return drop
def compute_tree(self, new_data=None):
"""
Computes the Merkle Tree associated with the corners in the block.
:param new_data: Corner list
:return: None
"""
if new_data is None:
self.merkle_tree = MerkleTree(self.corners, fast_hash)
else:
self.merkle_tree.extend(new_data)
def append_pgt_repro_data(drop: dict, data: dict):
merkletree = MerkleTree(data.items(), common_hash)
data['merkleroot'] = merkletree.merkle_root
# Separated so chaining can happen on independent elements (or both later)
drop['reprodata']['pgt_parenthashes'] = {}
drop['reprodata']['pgt_data'] = data
return drop
def newBlock(self,txn):
if(txn==[]):
return
Tno="Txn-"+str(blockChain.Tcount)
blockChain.allTxns[Tno]=txn[0]
blockChain.Tcount+=1
pH=blockChain.chain[-1].hash
PoW={}
PoW['Hash'],PoW['nonce']=self.PoW(pH,txn)
mT=MerkleTree(txn, HF)
mR=mT.merkle_root
#ID PROOF GENERATION
S = txn[0]['Sender']
txID = str(1)
diff=3
S.idProof['block_no'] = len(blockChain.chain)
S.idProof['block_hash'] = PoW['Hash']
S.idProof['difficulty'] = diff
S.idProof['txn_Index'] = txID
pk=S.pk
key=hmac.HMAC(pk)
S.idProof['Node_ID'] = hmac.HMAC(key.digest(),txID.encode()).hexdigest()
S.idProof['okToSend'] = '0'
S.idProof['okToRecieve'] = '0'
miner=blockChain.POC()
BLOC=block(pH,txn,PoW,mR,miner)
blockChain.chain.append(BLOC)
def addBlockToChain(self, blockchain, nonce, miner):
#adding fees to all transactions in block
for y, x in enumerate(self.transactions):
try:
self.transactions[y] = x.__dict__
except:
pass
for y, x in enumerate(self.transactions):
if (x["sender"] !=
"0000000000000000000000000000000000000000000000000000000000000000"
):
fee = calculateFee(self.transactions[y])
self.transactions[y]["outputs"][0] = [
self.transactions[y]["outputs"][0][0],
int(self.transactions[y]["txamount"] - fee)
]
self.transactions[y]["outputs"].append([miner, int(fee)])
#formatting for saving chain
self.miner = miner
self.nonce = hex(nonce)
self.tx_num = len(self.transactions)
#murkleTree = MerkleTree(self.transactions, stringHash)
self.mrkl = str(MerkleTree(self.transactions, stringHash))[12:-2]
blockchain.addBlock(self)
def init_lgt_repro_data(logical_graph_template: dict, rmode: str):
"""
Creates and appends graph-wide reproducibility data at the logical template stage.
Currently, this is basically a stub that adds the requested flag to the graph.
Later, this will contain significantly more information.
:param logical_graph_template: The logical graph data structure (a JSON object (a dict))
:param rmode: One several values 0-5 defined in constants.py
:return: The same lgt object with new information appended
"""
rmode = rflag_caster(rmode)
if not rmode_supported(rmode):
logger.warning("Requested reproducibility mode %s not yet implemented",
str(rmode))
rmode = REPRO_DEFAULT
if rmode == ReproducibilityFlags.NOTHING:
return logical_graph_template
reprodata = {
"rmode": str(rmode.value),
"meta_data": accumulate_meta_data()
}
meta_tree = MerkleTree(reprodata.items(), common_hash)
reprodata["merkleroot"] = meta_tree.merkle_root
for drop in logical_graph_template.get("nodeDataArray", []):
init_lgt_repro_drop_data(drop, rmode)
logical_graph_template["reprodata"] = reprodata
logger.info("Reproducibility data finished at LGT level")
return logical_graph_template
def get_chain_index():
chain_data = []
for block in blockchain.chain:
chain_data.append(block.__dict__)
node_chain ={
'length': len(chain_data),
'chain': chain_data,
'peers': list(peers)}
store_blockchain(node_chain)
dataa = json.dumps({"length": len(chain_data),
"chain": chain_data,
"peers": list(peers)})
retres = json.loads(dataa)
trans = []
i=0
for tr in blockchain.last_block.transactions:
print(tr)
i=i+1
trans_hash = generate_transaction_hash(tr)
trans.append({"index": i,"thash": trans_hash})
date = datetime.fromtimestamp(int(blockchain.last_block.timestamp))
merklroot = MerkleTree(trans)
data_tr ={"transactions":trans,"merkle":merklroot,"date":date}
return render_template("blockchain.html",blockchain=retres,inf=data_tr)
def agglomerate_leaves(leaves: list):
"""
Inserts all hash values in `leaves` into a merkleTree in sorted order (ascending).
Returns the root of this tree
"""
merkletree = MerkleTree(sorted(leaves))
return merkletree.merkle_root
def merkle_root(self):
"""
Returns a merkle root of the transaction hashes.
"""
tree = MerkleTree(self.transactions)
return tree.merkle_root
def build_pg_block_data(drop: dict):
block_data = [
drop['reprodata']['pg_data']['merkleroot'],
drop['reprodata']['pgt_blockhash'], drop['reprodata']['lg_blockhash']
]
for parenthash in sorted(drop['reprodata']['pg_parenthashes'].values()):
block_data.append(parenthash)
mtree = MerkleTree(block_data, common_hash)
drop['reprodata']['pg_blockhash'] = mtree.merkle_root
def _build_hash_payload(vertex: dict, data_fields, hash_function):
data = []
for key, val in vertex.items():
if data_fields is not None:
if key in data_fields:
data.append(val)
else:
data.append(val)
mtree = MerkleTree(data, hash_function)
return {"data_hash": mtree.merkle_root}
def make_tree(self, transactions):
"""
Make merkle tree from a list of transactions and return root hash
"""
tree = MerkleTree(transactions)
# beautify(tree)
stdev = self.standard_dev(transactions)
mean = self.mean(transactions)
variance = self.variance(transactions)
return tree.merkle_root, stdev, mean, variance
def createGenesis(self, miner, hashdemand):
self.chainDict = []
self.genesisHashDemand = hashdemand
genesis = block(self)
genesis.difficulty = "{:016x}".format(hashdemand)
genesis.complete(time.time())
genesis.mrkl = str(MerkleTree(genesis.transactions, stringHash))[12:-2]
genesis.tx_num = 1
mine(genesis, miner, self)
def _build_block_hash(data: dict, hash_function):
hashes = []
for _, val in data.items():
if val is not None:
if isinstance(val, collections.abc.Iterable):
hashes.extend(val)
else:
hashes.append(val)
mtree = MerkleTree(sorted(hashes), hash_function)
data["hash"] = mtree.merkle_root
def recompute_command(command, fout):
"""
Recomputes a given command, tracing the methods logged on the way.
These methods are loaded in a MerkleTree and returns its root.
:param command: The executed python command (string)
:param fout: The name of the file to store the trace
:return: A merkleroot of the trace file.
"""
np.random.seed(42) # Random number control
random.seed(42)
sys.stdout = open(fout, 'w')
tracer = trace.Trace(trace=1,
count=0)
tracer.run(command)
sys.stdout.close()
sys.stdout = sys.__stdout__
log = open(fout, 'r')
mtree = MerkleTree(log.readlines())
mtree.append(system_summary()['signature'])
return mtree.merkle_root
def init_pg_repro_drop_data(drop: dict):
"""
Creates and appends per-drop reproducibility information at the physical graph stage.
:param drop: The drop description
:return: The same drop with appended reproducibility information
"""
level = rflag_caster(drop["reprodata"]["rmode"])
data = accumulate_pg_drop_data(drop)
if level == ReproducibilityFlags.ALL:
for rmode in ALL_RMODES:
merkletree = MerkleTree(data[rmode.name].items(), common_hash)
data[rmode.name]["merkleroot"] = merkletree.merkle_root
drop["reprodata"][rmode.name]["pg_parenthashes"] = {}
drop["reprodata"][rmode.name]["pg_data"] = data[rmode.name]
else:
merkletree = MerkleTree(data.items(), common_hash)
data["merkleroot"] = merkletree.merkle_root
# Separated so chaining can happen on independent elements (or both later)
drop["reprodata"]["pg_parenthashes"] = {}
drop["reprodata"]["pg_data"] = data
return drop
def append_pgt_repro_data(drop: dict, data: dict):
"""
Adds provided data dictionary to drop description at PGT level.
:param drop: The drop description
:param data: The data to be added - arbitrary dictionary
:return:
"""
level = rflag_caster(drop["reprodata"]["rmode"])
if level == ReproducibilityFlags.ALL:
for rmode in ALL_RMODES:
merkletree = MerkleTree(data[rmode.name].items(), common_hash)
data[rmode.name]["merkleroot"] = merkletree.merkle_root
drop["reprodata"][rmode.name]["pgt_parenthashes"] = {}
drop["reprodata"][rmode.name]["pgt_data"] = data[rmode.name]
else:
merkletree = MerkleTree(data.items(), common_hash)
data["merkleroot"] = merkletree.merkle_root
# Separated so chaining can happen on independent elements (or both later)
drop["reprodata"]["pgt_parenthashes"] = {}
drop["reprodata"]["pgt_data"] = data
return drop
def __init__(self, scrooge_vk=None):
if self._sealed:
return
self._sealed = True
self._last_hash_pt = None
self._last_hash_pt_signed = None
self._users_coins = {}
self._users_coins[scrooge_vk] = []
self._merkle_tree = MerkleTree()
self._unconfirmed_transactions = []
def __init__(self,
blockchain=None,
name='block',
height=0,
version=0,
coinbase=None,
corners=None,
timestamp=0,
previous_hash=pycryptonight.cn_slow_hash(b''),
nonce=(0).to_bytes(NONCE_SIZE, 'big')):
super().__init__(f"{name} - {height} :")
self.blockchain = blockchain
self.version = version
self.timestamp = timestamp
self.previous_hash = previous_hash
self.nonce = nonce
self.coinbase = coinbase
self._corners = [] if corners is None else corners
self.merkle_tree = MerkleTree(self.corners, fast_hash)
self.hash = self.get_hash()
self.difficulty = 4
def main():
data = []
tree = MerkleTree(data, hashfunc)
for i in ascii_lowercase:
tree.append(i)
beautify(tree)
print("Original verify_leaf_inclusion")
original_proof = tree.get_proof('a')
print(tree.verify_leaf_inclusion('a', original_proof))
print("Proof to lists/ Lists to proof verify_leaf_inclusion")
root = tree._root
proof_hashs, proof_types = proof_to_lists(original_proof)
remade_proof = lists_to_proof(proof_hashs, proof_types)
print(
merklelib.verify_leaf_inclusion('a', remade_proof, hashfunc,
utils.to_hex(root.hash)))
print("Proof to string test")
string_proof = proof_to_string(original_proof)
print(type(string_proof) == type("hello"))
print("Proof to string/ String to proof verify_leaf_inclusion")
new_proof = string_to_proof(string_proof)
print(
merklelib.verify_leaf_inclusion('a', remade_proof, hashfunc,
utils.to_hex(root.hash)))
def init_pg_repro_drop_data(drop: dict):
"""
Creates and appends per-drop reproducibility information at the physical graph stage.
:param drop:
:return: The same drop with appended reproducibility information
"""
data = accumulate_pg_drop_data(drop)
merkletree = MerkleTree(data.items(), common_hash)
data['merkleroot'] = merkletree.merkle_root
# Separated so chaining can happen on independent elements (or both later)
drop['reprodata']['pg_parenthashes'] = {}
drop['reprodata']['pg_data'] = data
return drop
def __init__(self, pH, txnList, PoW, minerID):
self.blockHeight = block.blockCount
block.blockCount += 1
self.prevHash = pH
self.transactions = txnList
self.timestamp = DT.timestamp(DT.now())
self.hash = PoW['Hash']
self.nonce = PoW['nonce']
self.difficulty = PoW['difficulty']
self.mTree = MerkleTree(txnList)
self.mRoot = self.mTree.merkle_root
self.minerID = minerID
self.TransactionsLength = len(self.transactions)
self.TimeStamp = str(DT.fromtimestamp(self.timestamp))
def init_lgt_repro_drop_data(drop: dict, level: ReproducibilityFlags):
"""
Creates and appends per-drop reproducibility information at the logical template stage.
:param drop:
:param level:
:return: The same drop with appended reproducibility information.
"""
data = accumulate_lgt_drop_data(drop, level)
merkletree = MerkleTree(data.items(), common_hash)
data['merkleroot'] = merkletree.merkle_root
drop['reprodata'] = {
'rmode': str(level.value),
'lgt_data': data,
'lg_parenthashes': {}
}
return drop
def addNewBlock(self,txnList):
if(txnList==[]):
return
for T in txnList:
Tno="Txn-"+str(blockChain.Tcount)
blockChain.allTxns[Tno]=T
blockChain.Tcount+=1
pH=blockChain.chain[-1].hash
PoW={}
PoW['Hash'],PoW['nonce']=self.PoW(pH,txnList)
mT = MerkleTree(txnList, HF)
mR=mT.merkle_root
miner=blockChain.POC()
BLOC=block(pH,txnList,PoW,mR,miner)
blockChain.chain.append(BLOC)
