def get_previous_consensus_block_from_remote(self, block, peer):
# TODO: async conversion
retry = 0
while True:
try:
url = 'http://' + peer.to_string(
) + '/get-block?hash=' + block.prev_hash
if self.debug:
print('getting block', url)
res = requests.get(url,
timeout=1,
headers={'Connection': 'close'})
except:
if retry == 1:
raise BadPeerException()
else:
retry += 1
continue
try:
if self.debug:
print('response code: ', res.status_code)
new_block = Block.from_dict(
json.loads(res.content.decode('utf-8')))
if int(new_block.version) == CHAIN.get_version_for_height(
new_block.index):
return new_block
else:
return None
except:
return None
def get_latest_consensus_blocks(self):
for x in self.mongo.db.consensus.find({}, {
'_id': 0
}).sort([('index', -1)]):
if CHAIN.get_version_for_height(x['block']['index']) == int(
x['block']['version']):
yield x
async def get_next_consensus_block_from_local(self, block):
#table cleanup
new_block = await self.mongo.async_db.consensus.find_one({
'block.prevHash':
block.hash,
'block.index': (block.index + 1),
'block.version':
CHAIN.get_version_for_height((block.index + 1))
})
if new_block:
new_block = Block.from_dict(new_block['block'])
if int(new_block.version) == CHAIN.get_version_for_height(
new_block.index):
return new_block
else:
return None
return None
def get_consensus_blocks_by_index(self, index):
return self.mongo.db.consensus.find(
{
'index': index,
'block.prevHash': {
'$ne': ''
},
'block.version': CHAIN.get_version_for_height(index)
}, {'_id': 0})
async def get_previous_consensus_block_from_local(self, block, peer):
#table cleanup
new_block = await self.mongo.async_db.consensus.find_one({
'block.hash':
block.prev_hash,
'block.index': (block.index - 1),
'block.version':
CHAIN.get_version_for_height((block.index - 1)),
'ignore': {
'$ne': True
}
})
if new_block:
new_block = Block.from_dict(new_block['block'])
if int(new_block.version) == CHAIN.get_version_for_height(
new_block.index):
return new_block
else:
return None
return None
async def integrate_block_with_existing_chain(self,
block: Block,
extra_blocks=None):
"""Even in case of retrace, this is the only place where we insert a new block into the block collection and update BU"""
try:
# TODO: reorg the checks, to have the faster ones first.
# Like, here we begin with checking every tx one by one, when <e did not even check index and provided hash matched previous one.
try:
block.verify()
except Exception as e:
print("Integrate block error 1", e)
return False
for transaction in block.transactions:
try:
if extra_blocks:
transaction.extra_blocks = extra_blocks
transaction.verify()
except InvalidTransactionException as e:
print(e)
return False
except InvalidTransactionSignatureException as e:
print(e)
return False
except MissingInputTransactionException as e:
print(e)
return False
except NotEnoughMoneyException as e:
print(e)
return False
except Exception as e:
print(e)
return False
if block.index == 0:
return True
height = block.index
last_block = self.existing_blockchain.blocks[block.index - 1]
if last_block.index != (block.index -
1) or last_block.hash != block.prev_hash:
print("Integrate block error 2")
raise ForkException()
if not last_block:
print("Integrate block error 3")
raise ForkException()
target = BlockFactory.get_target(height, last_block, block,
self.existing_blockchain)
delta_t = int(time()) - int(last_block.time)
special_target = CHAIN.special_target(block.index, block.target,
delta_t,
get_config().network)
target_block_time = CHAIN.target_block_time(self.config.network)
if block.index >= 35200 and delta_t < 600 and block.special_min:
raise Exception('Special min block too soon')
# TODO: use a CHAIN constant for pow blocks limits
if ((int(block.hash, 16) < target) or
(block.special_min and int(block.hash, 16) < special_target)
or (block.special_min and block.index < 35200) or
(block.index >= 35200 and block.index < 38600
and block.special_min and
(int(block.time) - int(last_block.time)) > target_block_time)
):
if last_block.index == (
block.index -
1) and last_block.hash == block.prev_hash:
# self.mongo.db.blocks.update({'index': block.index}, block.to_dict(), upsert=True)
# self.mongo.db.blocks.remove({'index': {"$gt": block.index}}, multi=True)
# todo: is this useful? can we have more blocks above? No because if we had, we would have raised just above
await self.mongo.async_db.block.delete_many(
{'index': {
"$gte": block.index
}})
await self.mongo.async_db.blocks.replace_one(
{'index': block.index}, block.to_dict(), upsert=True)
# TODO: why do we need to keep that one in memory?
try:
self.existing_blockchain.blocks[block.index] = block
del self.existing_blockchain.blocks[block.index + 1:]
except:
self.existing_blockchain.blocks.append(block)
if self.debug:
self.app_log.info(
"New block inserted for height: {}".format(
block.index))
await self.config.on_new_block(
block) # This will propagate to BU
return True
else:
print("Integrate block error 4")
raise ForkException()
else:
print("Integrate block error 5")
raise AboveTargetException()
return False # unreachable code
except Exception as e:
exc_type, exc_obj, exc_tb = exc_info()
fname = path.split(exc_tb.tb_frame.f_code.co_filename)[1]
self.app_log.warning(
"integrate_block_with_existing_chain {} {} {}".format(
exc_type, fname, exc_tb.tb_lineno))
raise
async def sync_bottom_up(self):
try:
#bottom up syncing
last_latest = self.latest_block
self.latest_block = Block.from_dict(
await self.config.BU.get_latest_block_async())
if self.latest_block.index > last_latest.index:
self.app_log.info(
'Block height: %s | time: %s' %
(self.latest_block.index,
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
self.remove_pending_transactions_now_in_chain()
self.remove_fastgraph_transactions_now_in_chain()
latest_consensus = await self.mongo.async_db.consensus.find_one({
'index':
self.latest_block.index + 1,
'block.version':
CHAIN.get_version_for_height(self.latest_block.index + 1),
'ignore': {
'$ne': True
}
})
if latest_consensus:
latest_consensus = Block.from_dict(latest_consensus['block'])
if self.debug:
self.app_log.info("Latest consensus_block {}".format(
latest_consensus.index))
records = await self.mongo.async_db.consensus.find({
'index':
self.latest_block.index + 1,
'block.version':
CHAIN.get_version_for_height(self.latest_block.index + 1),
'ignore': {
'$ne': True
}
}).to_list(length=100)
for record in sorted(
records, key=lambda x: int(x['block']['target'], 16)):
await self.import_block(record)
last_latest = self.latest_block
self.latest_block = Block.from_dict(
await self.config.BU.get_latest_block_async())
if self.latest_block.index > last_latest.index:
self.app_log.info('Block height: %s | time: %s' % (
self.latest_block.index,
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
latest_consensus_now = await self.mongo.async_db.consensus.find_one(
{
'index':
self.latest_block.index + 1,
'block.version':
CHAIN.get_version_for_height(self.latest_block.index +
1),
'ignore': {
'$ne': True
}
})
if latest_consensus_now and latest_consensus.index == latest_consensus_now[
'index']:
await self.search_network_for_new()
return True
else:
await self.search_network_for_new()
return True
except Exception as e:
exc_type, exc_obj, exc_tb = exc_info()
fname = path.split(exc_tb.tb_frame.f_code.co_filename)[1]
self.app_log.warning("{} {} {}".format(exc_type, fname,
exc_tb.tb_lineno))
raise
def get_latest_consensus_block(self):
latests = self.get_latest_consensus_blocks()
for latest in latests:
if int(latest['block']['version']) == CHAIN.get_version_for_height(
latest['block']['index']):
return Block.from_dict(latest['block'])
"""
Temp. test for block reward function precision
"""
import sys
import random
sys.path.append('../')
from yadacoin.chain import CHAIN
if __name__ == "__main__":
for i in range(1000):
index = random.randint(1, 6930000+21000)
value1 = CHAIN.get_block_reward_deprecated(index)
value2 = CHAIN.get_block_reward(index)
if value1 != value2:
print("Error", index, value1, value2)
# boundary tests
test2 = (2519999, 2520000, 2520001, 2729999, 2730000, 2730001)
for index in test2:
value1 = CHAIN.get_block_reward_deprecated(index)
value2 = CHAIN.get_block_reward(index)
if value1 != value2:
print("Error2", index, value1, value2)
async def generate(cls,
config,
transactions,
public_key,
private_key,
force_version=None,
index=None,
force_time=None):
try:
mongo = config.mongo
app_log = getLogger("tornado.application")
if force_version is None:
version = CHAIN.get_version_for_height(index)
else:
version = force_version
if force_time:
xtime = str(int(force_time))
else:
xtime = str(int(time.time()))
index = int(index)
if index == 0:
prev_hash = ''
else:
prev_hash = config.BU.get_latest_block()['hash']
transaction_objs = []
fee_sum = 0.0
used_sigs = []
for txn in transactions:
try:
if isinstance(txn, FastGraph):
transaction_obj = txn
else:
transaction_obj = FastGraph.from_dict(index, txn)
if transaction_obj.transaction_signature in used_sigs:
print('duplicate transaction found and removed')
continue
if not transaction_obj.verify():
raise InvalidTransactionException(
"invalid transactions")
used_sigs.append(transaction_obj.transaction_signature)
except:
try:
if isinstance(txn, Transaction):
transaction_obj = txn
else:
transaction_obj = Transaction.from_dict(index, txn)
if transaction_obj.transaction_signature in used_sigs:
print('duplicate transaction found and removed')
continue
transaction_obj.verify()
used_sigs.append(transaction_obj.transaction_signature)
except:
raise InvalidTransactionException(
"invalid transactions")
try:
if int(index) > CHAIN.CHECK_TIME_FROM and (
int(transaction_obj.time) >
int(xtime) + CHAIN.TIME_TOLERANCE):
app_log.debug("Block embeds txn too far in the future")
continue
transaction_objs.append(transaction_obj)
fee_sum += float(transaction_obj.fee)
except Exception as e:
await mongo.async_db.miner_transactions.delete_many(
{'id': transaction_obj.transaction_signature})
if config.debug:
app_log.debug('Exception {}'.format(e))
else:
continue
block_reward = CHAIN.get_block_reward(index)
coinbase_txn_fctry = TransactionFactory(
index,
public_key=public_key,
private_key=private_key,
outputs=[{
'value':
block_reward + float(fee_sum),
'to':
str(
P2PKHBitcoinAddress.from_pubkey(
bytes.fromhex(public_key)))
}],
coinbase=True)
coinbase_txn = coinbase_txn_fctry.generate_transaction()
transaction_objs.append(coinbase_txn)
transactions = transaction_objs
block_factory = cls()
block = Block(version=version,
block_time=xtime,
block_index=index,
prev_hash=prev_hash,
transactions=transactions,
public_key=public_key)
txn_hashes = block.get_transaction_hashes()
block.set_merkle_root(txn_hashes)
block.merkle_root = block.verify_merkle_root
block_factory.block = block
return block_factory
except Exception as e:
import sys, os
print("Exception {} BlockFactory".format(e))
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
raise
def verify(self):
try:
getcontext().prec = 8
if int(self.version) != int(
CHAIN.get_version_for_height(self.index)):
raise Exception("Wrong version for block height", self.version,
CHAIN.get_version_for_height(self.index))
txns = self.get_transaction_hashes()
self.set_merkle_root(txns)
if self.verify_merkle_root != self.merkle_root:
raise Exception("Invalid block merkle root")
header = BlockFactory.generate_header(self)
hashtest = BlockFactory.generate_hash_from_header(
header, str(self.nonce))
# print("header", header, "nonce", self.nonce, "hashtest", hashtest)
if self.hash != hashtest:
getLogger("tornado.application").warning(
"Verify error hashtest {} header {} nonce {}".format(
hashtest, header, self.nonce))
raise Exception('Invalid block hash')
address = P2PKHBitcoinAddress.from_pubkey(
bytes.fromhex(self.public_key))
try:
# print("address", address, "sig", self.signature, "pubkey", self.public_key)
result = verify_signature(base64.b64decode(self.signature),
self.hash.encode('utf-8'),
bytes.fromhex(self.public_key))
if not result:
raise Exception("block signature1 is invalid")
except:
try:
result = VerifyMessage(
address,
BitcoinMessage(self.hash.encode('utf-8'), magic=''),
self.signature)
if not result:
raise
except:
raise Exception("block signature2 is invalid")
# verify reward
coinbase_sum = 0
for txn in self.transactions:
if int(self.index) > CHAIN.CHECK_TIME_FROM and (int(
txn.time) > int(self.time) + CHAIN.TIME_TOLERANCE):
raise Exception("Block embeds txn too far in the future")
if txn.coinbase:
for output in txn.outputs:
coinbase_sum += float(output.value)
fee_sum = 0.0
for txn in self.transactions:
if not txn.coinbase:
fee_sum += float(txn.fee)
reward = CHAIN.get_block_reward(self.index)
#if Decimal(str(fee_sum)[:10]) != Decimal(str(coinbase_sum)[:10]) - Decimal(str(reward)[:10]):
"""
KO for block 13949
0.02099999 50.021 50.0
Integrate block error 1 ('Coinbase output total does not equal block reward + transaction fees', 0.020999999999999998, 0.021000000000000796)
"""
if quantize_eight(fee_sum) != quantize_eight(coinbase_sum -
reward):
print(fee_sum, coinbase_sum, reward)
raise Exception(
"Coinbase output total does not equal block reward + transaction fees",
fee_sum, (coinbase_sum - reward))
except Exception as e:
exc_type, exc_obj, exc_tb = exc_info()
fname = path.split(exc_tb.tb_frame.f_code.co_filename)[1]
getLogger("tornado.application").warning("verify {} {} {}".format(
exc_type, fname, exc_tb.tb_lineno))
raise
def get_target(cls, height, last_block, block, blockchain) -> int:
try:
# change target
max_target = CHAIN.MAX_TARGET
if get_config().network in ['regnet', 'testnet']:
return int(max_target)
max_block_time = CHAIN.target_block_time(get_config().network)
retarget_period = CHAIN.RETARGET_PERIOD # blocks
max_seconds = CHAIN.TWO_WEEKS # seconds
min_seconds = CHAIN.HALF_WEEK # seconds
if height >= CHAIN.POW_FORK_V3:
retarget_period = CHAIN.RETARGET_PERIOD_V3
max_seconds = CHAIN.MAX_SECONDS_V3 # seconds
min_seconds = CHAIN.MIN_SECONDS_V3 # seconds
elif height >= CHAIN.POW_FORK_V2:
retarget_period = CHAIN.RETARGET_PERIOD_V2
max_seconds = CHAIN.MAX_SECONDS_V2 # seconds
min_seconds = CHAIN.MIN_SECONDS_V2 # seconds
if height > 0 and height % retarget_period == 0:
get_config().debug_log(
"RETARGET get_target height {} - last_block {} - block {}/time {}"
.format(height, last_block.index, block.index, block.time))
block_from_2016_ago = Block.from_dict(
get_config().BU.get_block_by_index(height -
retarget_period))
get_config().debug_log(
"Block_from_2016_ago - block {}/time {}".format(
block_from_2016_ago.index, block_from_2016_ago.time))
two_weeks_ago_time = block_from_2016_ago.time
elapsed_time_from_2016_ago = int(
last_block.time) - int(two_weeks_ago_time)
get_config().debug_log(
"elapsed_time_from_2016_ago {} s {} days".format(
int(elapsed_time_from_2016_ago),
elapsed_time_from_2016_ago / (60 * 60 * 24)))
# greater than two weeks?
if elapsed_time_from_2016_ago > max_seconds:
time_for_target = max_seconds
get_config().debug_log("gt max")
elif elapsed_time_from_2016_ago < min_seconds:
time_for_target = min_seconds
get_config().debug_log("lt min")
else:
time_for_target = int(elapsed_time_from_2016_ago)
block_to_check = last_block
if blockchain.partial:
start_index = len(blockchain.blocks) - 1
else:
start_index = last_block.index
get_config().debug_log("start_index {}".format(start_index))
while 1:
if block_to_check.special_min or block_to_check.target == max_target or not block_to_check.target:
block_to_check = blockchain.blocks[start_index]
start_index -= 1
else:
target = block_to_check.target
break
get_config().debug_log("start_index2 {}, target {}".format(
block_to_check.index,
hex(int(target))[2:].rjust(64, '0')))
new_target = int((time_for_target * target) / max_seconds)
get_config().debug_log("new_target {}".format(
hex(int(new_target))[2:].rjust(64, '0')))
if new_target > max_target:
target = max_target
else:
target = new_target
elif height == 0:
target = max_target
else:
block_to_check = block
delta_t = int(block.time) - int(last_block.time)
if block.index >= 38600 and delta_t > max_block_time and block.special_min:
special_target = CHAIN.special_target(
block.index, block.target, delta_t,
get_config().network)
return special_target
block_to_check = last_block # this would be accurate. right now, it checks if the current block is under its own target, not the previous block's target
if blockchain.partial:
start_index = len(blockchain.blocks) - 1
else:
start_index = last_block.index
while 1:
if start_index == 0:
return block_to_check.target
if block_to_check.special_min or block_to_check.target == max_target or not block_to_check.target:
block_to_check = blockchain.blocks[start_index]
start_index -= 1
else:
target = block_to_check.target
break
return int(target)
except Exception as e:
import sys, os
print("Exception {} get_target".format(e))
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
raise