def handle_orphans() -> "bool":
"""
tries to attach an orphan block to the head of the primary or secondary blockchain.
Sometimes blocks are received out of order and cannot be attached to the primary
or secondary blockchains. These blocks are placed in an orphan_blocks list and as
new blocks are added to the primary or secondary blockchains, an attempt is made to
add orphaned blocks to the blockchain(s).
"""
try:
# iterate through the orphan blocks attempting to append an orphan
# block to a blockchain
if len(bchain.blockchain) == 0: return
# try to append to the primary blockchain
for block in orphan_blocks:
if block['prevblockhash'] == bchain.blockheader_hash(
bchain.blockchain[-1]):
if block["height"] == bchain.blockchain[-1]["height"] + 1:
bchain.blockchain.append(block)
orphan_blocks.remove(block)
# remove this orphan blocks transactions from the mempool
with semaphore:
remove_mempool_transactions(block)
# try to append to the secondary blockchain
if len(bchain.secondary_blockchain) > 0:
for block in orphan_blocks:
if block['prevblockhash'] == bchain.blockheader_hash(
bchain.secondary_blockchain[-1]):
if block["height"] == bchain.secondary_blockchain[-1][
"height"] + 1:
bchain.secondary_blockchain.append(block)
orphan_blocks.remove(block)
# remove this blocks transactions from the mempool
with semaphore:
remove_mempool_transactions(block)
# remove stale orphaned blocks
for block in orphan_blocks:
if bchain.blockchain[-1]["height"] >= 3:
if bchain.blockchain[-1]["height"] - block["height"] >= 2:
orphan_blocks.remove(block)
except Exception as err:
logging.debug('handle_orphans: exception: ' + str(err))
return False
return True
def test_merkle_root_block_transaction(monkeypatch):
"""
test merkle root for synthetic random transactions
"""
hblockchain.blockchain.clear()
block_0["merkle_root"] = hblockchain.merkle_root(block_0["tx"], True)
assert hblockchain.add_block(block_0) == True
block_1["merkle_root"] = hblockchain.merkle_root(block_1["tx"], True)
block_1["prevblockhash"] = hblockchain.blockheader_hash(block_0)
assert hblockchain.add_block(block_1) == True
block_2["merkle_root"] = hblockchain.merkle_root(block_2["tx"], True)
block_2["prevblockhash"] = hblockchain.blockheader_hash(block_1)
assert hblockchain.add_block(block_2) == True
def test_swap_blockchain():
"""
test swap the primary and secondary blockchains
"""
hblockchain.blockchain.clear()
hblockchain.secondary_blockchain.clear()
hmining.received_blocks.clear()
block1 = make_synthetic_block()
block2 = make_synthetic_block()
block3 = make_synthetic_block()
block4 = make_synthetic_block()
block4["prevblockhash"] = hblockchain.blockheader_hash(block3)
hblockchain.blockchain.append(block1)
hblockchain.secondary_blockchain.append(block2)
hblockchain.secondary_blockchain.append(block3)
hblockchain.secondary_blockchain.append(block4)
assert len(hblockchain.blockchain) == 1
assert len(hblockchain.secondary_blockchain) == 3
hmining.swap_blockchains()
assert len(hblockchain.blockchain) == 3
assert len(hblockchain.secondary_blockchain) == 1
hblockchain.blockchain.clear()
hblockchain.secondary_blockchain.clear()
def test_append_to_primary_blockchain(monkeypatch):
"""
test add a received block to the primary blockchain
"""
monkeypatch.setattr(hblockchain, "validate_block", lambda x: True)
monkeypatch.setattr(tx, "validate_transaction", lambda x,y: True)
monkeypatch.setattr(hchaindb, "get_transaction", lambda x: True)
monkeypatch.setattr(hchaindb, "transaction_update", lambda x: True)
hblockchain.blockchain.clear()
hblockchain.secondary_blockchain.clear()
hmining.received_blocks.clear()
block1 = make_synthetic_block()
block2 = make_synthetic_block()
block3 = make_synthetic_block()
block4 = make_synthetic_block()
block4["prevblockhash"] = hblockchain.blockheader_hash(block3)
hblockchain.blockchain.append(block1)
hblockchain.blockchain.append(block2)
hblockchain.blockchain.append(block3)
assert len(hblockchain.blockchain) == 3
hmining.received_blocks.append(block4)
assert len(hmining.received_blocks) == 1
hmining.process_received_blocks()
assert len(hblockchain.blockchain) == 4
assert len(hblockchain.secondary_blockchain) == 0
hblockchain.blockchain.clear()
hblockchain.secondary_blockchain.clear()
hmining.received_blocks.clear()
def make_synthetic_blocks(num_blocks):
"""
make synthetic blocks for unit testing
add the blocks to the blockchain
"""
for ctr in range(num_blocks):
block = {}
block["transactionid"] = rcrypt.make_uuid()
block["version"] = hconfig.conf["VERSION_NO"]
block["difficulty_bits"] = hconfig.conf["DIFFICULTY_BITS"]
block["nonce"] = hconfig.conf["NONCE"]
block["height"] = ctr
block["timestamp"] = int(time.time())
block["tx"] = []
# add transactions to the block
num_tx = secrets.randbelow(5) + 1
for __ctr in range(num_tx):
trx = make_synthetic_transaction(block)
block["tx"].append(trx)
block["merkle_root"] = hblockchain.merkle_root(block["tx"], True)
if ctr == 0:
block["prevblockhash"] = ""
else:
block["prevblockhash"] = hblockchain.blockheader_hash(
hblockchain.blockchain[ctr - 1])
# append block to blockchain
hblockchain.blockchain.append(block)
return
def make_blocks(num_blocks):
ctr = 0
total_tx = 0
blocks = []
global unspent_fragments
unspent_fragments.clear()
hblockchain.blockchain.clear()
while ctr < num_blocks:
block = {}
block["prevblockhash"] = ""
block["version"] = hconfig.conf["VERSION_NO"]
block["timestamp"] = int(time.time())
block["difficulty_bits"] = hconfig.conf["DIFFICULTY_BITS"]
block["nonce"] = hconfig.conf["NONCE"]
block["merkle_root"] = ""
block["height"] = ctr
block["tx"] = []
# make a random number of transactions for this block
# genesis block is ctr == 0
if ctr == 0: num_transactions = 200
else:
num_transactions = secrets.randbelow(50)
if num_transactions <= 1: num_transactions = 25
txctr = 0
while txctr < num_transactions:
if ctr > 0 and txctr == 0: coinbase_trans = True
else: coinbase_trans = False
trx = make_random_transaction(ctr, coinbase_trans)
assert trx != False
block["tx"].append(trx)
total_tx += 1
txctr += 1
if ctr > 0:
block["prevblockhash"] = hblockchain.blockheader_hash(hblockchain.blockchain[ctr - 1])
ret = hblockchain.merkle_root(block["tx"], True)
assert ret != False
block["merkle_root"] = ret
ret = hblockchain.add_block(block)
assert ret == True
blocks.append(block)
ctr+= 1
print("blockchain height: " + str(blocks[-1]["height"]))
print("total transactions count: " + str(total_tx))
return blocks
def make_blocks(num_blocks):
ctr = 0
blocks = []
global unspent_fragments
unspent_fragments.clear()
hblockchain.blockchain.clear()
while ctr < num_blocks:
block = {
"prevblockhash": "",
"version": "1",
"timestamp": int(time.time()),
"difficulty_bits": 20,
"nonce": 0,
"merkle_root": "",
"height": ctr,
"tx": []
}
# make a random number of transactions for this block
# genesis block is ctr == 0
if ctr == 0: num_transactions = 200
else:
num_transactions = secrets.randbelow(50)
if num_transactions == 0: num_transactions = 40
num_transactions = 2
txctr = 0
while txctr < num_transactions:
if ctr > 0 and txctr == 0: is_coinbase = True
else: is_coinbase = False
trx = make_random_transaction(ctr, is_coinbase)
assert trx != False
block["tx"].append(trx)
txctr += 1
if ctr > 0:
block["prevblockhash"] = hblockchain.blockheader_hash(
hblockchain.blockchain[ctr - 1])
ret = hblockchain.merkle_root(block["tx"], True)
assert ret != False
block["merkle_root"] = ret
ret = hblockchain.add_block(block)
assert ret == True
blocks.append(block)
ctr += 1
return blocks
async def mine_block(candidate_block: 'dictionary') -> "bool":
"""
Mines a candidate block.
Returns the solution nonce as a hexadecimal string if the block is
mined and False otherwise
Executes in a Python thread
"""
try:
final_nonce = None
save_block = dict(candidate_block)
# Loop until block is mined
while True:
# compute the SHA-256 hash for the block header of the candidate block
hash = bchain.blockheader_hash(candidate_block)
# convert the SHA-256 hash string to a Python integer
mined_value = int(hash, 16)
mined_value = 1 / mined_value
# test to determine whether the block has been mined
if mined_value < hconfig.conf["DIFFICULTY_NUMBER"]:
final_nonce = candidate_block["nonce"]
break
with semaphore:
if len(received_blocks) > 0:
if compare_transaction_lists(candidate_block) == False:
return False
# failed to mine the block so increment the
# nonce and try again
candidate_block['nonce'] += 1
logging.debug('mining.py: block has been mined')
# add block to the miner's blockchain
with semaphore:
ret = bchain.add_block(save_block)
if ret == False:
raise (ValueError("failed to add mined block to blockchain"))
# propagate the block on the Helium network
propagate_mined_block(candidate_block)
except Exception as err:
logging.debug('mine_block: exception: ' + str(err))
return False
return hex(final_nonce)
def test_no_consecutive_duplicate_blocks(monkeypatch):
"""
test cannot add the same block twice consecutively to the blockchain
"""
hblockchain.blockchain.clear()
monkeypatch.setattr(hblockchain, "merkle_root",
lambda x, y: rcrypt.make_SHA256_hash('msg0'))
assert hblockchain.add_block(block_0) == True
monkeypatch.setattr(hblockchain, "merkle_root",
lambda x, y: rcrypt.make_SHA256_hash('msg1'))
monkeypatch.setitem(block_1, "prevblockhash",
hblockchain.blockheader_hash(block_0))
assert hblockchain.add_block(block_1) == True
monkeypatch.setitem(block_1, "height", 2)
assert hblockchain.add_block(block_1) == False
hblockchain.blockchain.clear()
def test_read_second_block(monkeypatch):
"""
test reading the second block from the blockchain
"""
hblockchain.blockchain.clear()
assert len(hblockchain.blockchain) == 0
monkeypatch.setattr(hblockchain, "merkle_root",
lambda x, y: rcrypt.make_SHA256_hash('msg0'))
monkeypatch.setitem(block_1, "prevblockhash",
hblockchain.blockheader_hash(block_0))
ret = hblockchain.add_block(block_0)
assert ret == True
monkeypatch.setattr(hblockchain, "merkle_root", lambda x, y: \
rcrypt.make_SHA256_hash('msg1'))
ret = hblockchain.add_block(block_1)
assert ret == True
block = hblockchain.read_block(1)
assert block != False
hblockchain.blockchain.clear()
def test_add_orphan_to_secondary_blockchain(monkeypatch):
"""
test to add orphan block to the secondary blockchain
"""
hmining.orphan_blocks.clear()
hblockchain.blockchain.clear()
hblockchain.secondary_blockchain.clear()
hmining.received_blocks.clear()
#monkeypatch.setattr(hblockchain, "validate_block", lambda x: True)
monkeypatch.setattr(tx, "validate_transaction", lambda x, y: True)
monkeypatch.setattr(hchaindb, "transaction_update", lambda x: True)
monkeypatch.setattr(blk_index, "put_index", lambda x, y: True)
block0 = make_synthetic_block()
hblockchain.blockchain.append(block0)
block1 = make_synthetic_block()
block1["height"] = 1290
block2 = make_synthetic_block()
block2["height"] = 1291
block3 = make_synthetic_block()
block3["height"] = 1292
block3["prevblockhash"] = hblockchain.blockheader_hash(block2)
hblockchain.secondary_blockchain.append(block1)
hblockchain.secondary_blockchain.append(block2)
assert len(hblockchain.secondary_blockchain) == 2
hmining.orphan_blocks.append(block3)
assert len(hmining.orphan_blocks) == 1
hmining.handle_orphans()
assert len(hmining.orphan_blocks) == 0
assert len(hblockchain.secondary_blockchain) == 1
assert len(hblockchain.blockchain) == 3
hblockchain.blockchain.clear()
hblockchain.secondary_blockchain.clear()
hmining.orphan_blocks.clear()
def test_block_height(monkeypatch):
"""
test height of the the second block
"""
hblockchain.blockchain.clear()
monkeypatch.setattr(hblockchain, "merkle_root",
lambda x, y: rcrypt.make_SHA256_hash('msg0'))
monkeypatch.setitem(block_0, "height", 0)
monkeypatch.setitem(block_0, "prevblockhash", "")
monkeypatch.setitem(block_1, "height", 1)
monkeypatch.setitem(block_1, "prevblockhash",
hblockchain.blockheader_hash(block_0))
assert hblockchain.add_block(block_0) == True
monkeypatch.setattr(hblockchain, "merkle_root",
lambda x, y: rcrypt.make_SHA256_hash('msg1'))
assert hblockchain.add_block(block_1) == True
blk = hblockchain.read_block(1)
assert blk != False
assert blk["height"] == 1
hblockchain.blockchain.clear()
def proof_of_work(block):
"""
Proves whether a received block has in fact been mined.
Returns True or False
"""
try:
if block['difficulty_bits'] != hconfig.conf["DIFFICULTY_BITS"]:
raise (ValueError("wrong difficulty bits used"))
# compute the SHA-256 hash for the block header of the candidate block
hash = bchain.blockheader_hash(block)
# convert the SHA-256 hash string to a Python integer to base 10
mined_value = int(hash, 16)
mined_value = 1 / mined_value
# test to determine whether the block has been mined
if mined_value < hconfig.conf["DIFFICULTY_NUMBER"]: return True
except Exception as err:
logging.debug('proof_of_work: exception: ' + str(err))
return False
return False
def process_received_blocks() -> 'bool':
"""
processes mined blocks that are in the in the received_blocks
list and attempts to add these blocks to a blockchain.
Algorithm:
(1) get a block from the received blocks list.
(2) if the blockchain is empty and the block has a empty prevblockhash field
then add the block to the blockchain.
(3) Compute the block header hash of the last block on the blockchain (blkhash).
if blkhash == block["prevblockhash"] then add the block to the blockchain.
(4) if the blockchain has at least two blocks, then if:
let blk_hash be the hash second-latest block of the blockchain, then if
block["prevblockhash"] == blk_hash
create a secondary block consisting of all of the blocks of the blockchain
except for the latest. Append the block to the secondary blockchain.
(5) Otherwise move the block to the orphans list.
(6) If the received block was attached to a blockchain, for each block in the orphans
list, try to attach the orphan block to the primary blockchain or the secondary
blockchain if it exists.
(7) If the received block was attached to a blockchain and the secondary blockchain has
elements then swap the blockchain and the secondary blockchain if the length of the
secondary blockchain is greater.
(8) if the receive block was attached and the secondary blockchain has elements then
clear the secondary blockchain if its length is more than two blocks behind the
primary blockchain.
Note: Runs In A Python thread
"""
while True:
# process all of the blocks in the received blocks list
add_flag = False
# get a received block
if len(received_blocks) > 0:
block = received_blocks.pop()
else:
return True
# add it to the primary blockchain
with semaphore:
if bchain.add_block(block) == True:
logging.debug(
'receive_mined_block: block added to primary blockchain')
add_flag = True
# test whether the primary blockchain must be forked
# if the previous block hash is equal to the hash of the parent block of the
# block at the head of the blockchain add the block as a child of the parent and
# create a secondary blockchain. This constitutes a fork of the primary
# blockchain
elif len(bchain.blockchain) >= 2 and block[
'prevblockhash'] == bchain.blockheader_hash(
bchain.blockchain[-2]):
logging.debug('receive_mined_block: forking the blockchain')
fork_blockchain(block)
if bchain.add_block(block) == True:
logging.debug(
'receive_mined_block: block added to blockchain')
add_flag = True
# add it to the secondary blockchain
elif len(bchain.secondary_blockchain) > 0 and block[
'prevblockhash'] == bchain.blockheader_hash(
bchain.secondary_blockchain[-1]):
swap_blockchains()
if bchain.add_block(block) == True:
logging.debug(
'receive_mined_block: block added to blockchain')
add_flag = True
# cannot attach the block to a blockchain, place it in the orphans list
else:
orphan_blocks.append(block)
if add_flag == True:
if block["height"] % hconfig.conf["RETARGET_INTERVAL"] == 0:
retarget_difficulty_number(block)
handle_orphans()
swap_blockchains()
# remove any transactions in this block that are also
# in the the mempool
with semaphore:
remove_mempool_transactions(block)
propagate_mined_block(block)
return True
def test_computes_previous_block_hash(monkeypatch):
"""
test previous block hash has correct format
"""
val = hblockchain.blockheader_hash(block_0)
rcrypt.validate_SHA256_hash(val) == True
#############################################
# Build Three Synthetic Blocks For Testing
#############################################
block_0 = {
"prevblockhash": "",
"version": "1",
"timestamp": 0,
"difficulty_bits": 20,
"nonce": 0,
"merkle_root": rcrypt.make_SHA256_hash('msg0'),
"height": 0,
"tx": [make_random_transaction(0)]
}
block_1 = {
"prevblockhash": hblockchain.blockheader_hash(block_0),
"version": "1",
"timestamp": 0,
"difficulty_bits": 20,
"nonce": 0,
"merkle_root": rcrypt.make_SHA256_hash('msg1'),
"height": 1,
}
block_1["tx"] = []
block_1["tx"].append(make_random_transaction(1))
block_1["tx"].append(make_random_transaction(1))
block_2 = {
"prevblockhash": hblockchain.blockheader_hash(block_1),
"version": "1",
"timestamp": 0,
def test_clear_blockchain(monkeypatch):
"""
builds and clears the primary and secondary blockchains of a node
"""
# clear the primary and secondary blockchains for this test
ret = networknode.hclient(
"http://127.0.0.51:8081",
'{"jsonrpc":"2.0", "method": "clear_blockchain", "params": {}, "id": 19}'
)
assert ret.find("ok") >= 0
monkeypatch.setattr(tx, "validate_transaction", lambda x, y: True)
num_blocks = 20
blocks = make_blocks(num_blocks)
assert len(blocks) == num_blocks
tmp = hblockchain.blockheader_hash(blocks[0])
assert tmp == blocks[1]["prevblockhash"]
assert blocks[1]["height"] == 1
rpc = json.dumps({
"jsonrpc": "2.0",
"method": "receive_block",
"params": {
"block": blocks[0]
},
"id": 21
})
ret = networknode.hclient("http://127.0.0.51:8081", rpc)
assert ret.find("ok") != -1
ret = networknode.hclient(
"http://127.0.0.51:8081",
'{"jsonrpc": "2.0", "method": "get_blockchain_height", "params": {}, "id": 22}'
)
assert ret.find("error") == -1
height = (json.loads(ret))["result"]
# value of height attribute of the latest block in the blockchain
# meaning there is one block in the blockchain
assert height == 0
rpc = json.dumps({
"jsonrpc": "2.0",
"method": "receive_block",
"params": {
"block": blocks[1]
},
"id": 23
})
ret = networknode.hclient("http://127.0.0.51:8081", rpc)
assert ret.find("ok") != -1
ret = networknode.hclient(
"http://127.0.0.51:8081",
'{"jsonrpc": "2.0", "method": "get_blockchain_height", "params": {}, "id": 24}'
)
assert ret.find("error") == -1
height = (json.loads(ret))["result"]
# value of height attribute of the latest block in the blockchain
# meaning there are two blocks in the blockchain
assert height == 1
# clear the primary and secondary blockchains for this test
ret = networknode.hclient(
"http://127.0.0.51:8081",
'{"jsonrpc":"2.0", "method": "clear_blockchain", "params": {}, "id": 25}'
)
assert ret.find("ok") >= 0
async def make_candidate_block() -> "dictionary || bool":
"""
makes a candidate block for inclusion in the Helium blockchain.
A candidate block is created by:
(i) fetching transactions from the mempool and adding them to
the candidate blocks transaction list.
(ii) specifying the block header.
returns the candidate block or returns False if there is an error or if
the mempool is empty.
Executes in a Python thread
"""
try:
# if the mempool is empty then no transactions can be put into
# the candidate block
if len(mempool) == 0: return False
# make a public-private key pair that the miner will use to receive
# the mining reward as well as the transaction fees.
key_pair = make_miner_keys()
block = {}
# create a incomplete candidate block header
block['version'] = hconfig.conf["VERSION_NO"]
block['timestamp'] = int(time.time())
block['difficulty_bits'] = hconfig.conf["DIFFICULTY_BITS"]
block['nonce'] = hconfig.conf["NONCE"]
if len(bchain.blockchain) > 0:
block['height'] = bchain.blockchain[-1]["height"] + 1
else:
block['height'] = 0
block['merkle_root'] = ""
# get the value of the hash of the previous block's header
# this induces tamperproofness for the blockchain
if len(bchain.blockchain) > 0:
block['prevblockhash'] = bchain.blockheader_hash(
bchain.blockchain[-1])
else:
block['prevblockhash'] = ""
# calculate the size (in bytes) of the candidate block header
# The number 64 is the byte size of the SHA-256 hexadecimal
# merkle root. The merkle root is computed after all the
# transactions are included in the candidate block
# reserve 1000 bytes for the coinbase transaction
block_size = sys.getsizeof(block['version'])
block_size += sys.getsizeof(block['timestamp'])
block_size += sys.getsizeof(block['difficulty_bits'])
block_size += sys.getsizeof(block['nonce'])
block_size += sys.getsizeof(block['height'])
block_size += sys.getsizeof(block['prevblockhash'])
block_size += 64
block_size += sys.getsizeof(block['timestamp'])
block_size += 1000
# list of transactions in the block
block['tx'] = []
# get the Unix Time now
now = int(time.time())
# add transactions from the mempool to the candidate block until
# the transactions in the mempool are exhausted or the block
# attains it's maximum permissible size
for memtx in mempool:
# do not process future transactions
if memtx['locktime'] > now: continue
memtx = add_transaction_fee(memtx, key_pair[1])
# add the transaction to the candidate block
block_size += sys.getsizeof(memtx)
if block_size <= hconfig.conf['MAX_BLOCK_SIZE']:
block['tx'].append(memtx)
remove_list.append(memtx)
else:
break
# return if there are no transactions in the block
if len(block["tx"]) == 0: return False
# add a coinbase transaction
coinbase_tx = make_coinbase_transaction(block['height'], key_pair[1])
block['tx'].insert(0, coinbase_tx)
# update the length of the block
block_size += sys.getsizeof(block['tx'])
# calculate the merkle root of this block
ret = bchain.merkle_root(block['tx'], True)
if ret == False:
logging.debug('mining::make_candidate_block - merkle root error')
return False
block['merkle_root'] = ret
###################################
# validate the candidate block
###################################
if bchain.validate_block(block) == False:
logging.debug(
'mining::make_candidate_block - invalid block header')
return False
except Exception as err:
logging.debug('make_candidate_block: exception: ' + str(err))
# At this stage the candidate block has been created and it can be mined
return block