def create_block_to_mine(self,
m_header_list,
address=None,
create_time=None):
if not address:
address = Address.create_empty_account()
if create_time is None:
create_time = max(self.tip.create_time + 1, int(time.time()))
tracking_data = {
"inception": time_ms(),
"cluster": self.env.cluster_config.MONITORING.CLUSTER_ID,
}
difficulty = self.diff_calc.calculate_diff_with_parent(
self.tip, create_time)
block = self.tip.create_block_to_append(create_time=create_time,
address=address,
difficulty=difficulty)
block.minor_block_header_list = m_header_list
coinbase_amount = self.env.quark_chain_config.ROOT.COINBASE_AMOUNT
reward_tax_rate = self.env.quark_chain_config.reward_tax_rate
# the ratio of minor block coinbase
ratio = (1 - reward_tax_rate) / reward_tax_rate # type: Fraction
minor_block_fee = 0
for header in m_header_list:
minor_block_fee += header.coinbase_amount
# note the minor block fee is after tax
coinbase_amount += minor_block_fee * ratio.denominator // ratio.numerator
tracking_data["creation_ms"] = time_ms() - tracking_data["inception"]
block.tracking_data = json.dumps(tracking_data).encode("utf-8")
return block.finalize(coinbase_amount=coinbase_amount,
coinbase_address=address)
def create_block_to_mine(self,
m_header_list,
address=None,
create_time=None):
if not address:
address = Address.create_empty_account()
if create_time is None:
create_time = max(self.tip.create_time + 1, int(time.time()))
tracking_data = {
"inception": time_ms(),
"cluster": self.env.cluster_config.MONITORING.CLUSTER_ID,
}
difficulty = self.diff_calc.calculate_diff_with_parent(
self.tip, create_time)
block = self.tip.create_block_to_append(create_time=create_time,
address=address,
difficulty=difficulty)
block.minor_block_header_list = m_header_list
coinbase_tokens = self._calculate_root_block_coinbase(
[header.get_hash() for header in m_header_list],
block.header.height)
tracking_data["creation_ms"] = time_ms() - tracking_data["inception"]
block.tracking_data = json.dumps(tracking_data).encode("utf-8")
return block.finalize(coinbase_tokens=coinbase_tokens,
coinbase_address=address)
def create_block_to_mine(self, m_header_list, address, create_time=None):
if create_time is None:
create_time = max(self.tip.create_time + 1, int(time.time()))
extra_data = {
"inception": time_ms(),
"cluster": self.env.cluster_config.MONITORING.CLUSTER_ID,
}
difficulty = self.diff_calc.calculate_diff_with_parent(
self.tip, create_time)
block = self.tip.create_block_to_append(create_time=create_time,
address=address,
difficulty=difficulty)
block.minor_block_header_list = m_header_list
coinbase_amount = 0
for header in m_header_list:
coinbase_amount += header.coinbase_amount
coinbase_amount = coinbase_amount // 2
extra_data["creation_ms"] = time_ms() - extra_data["inception"]
block.header.extra_data = json.dumps(extra_data).encode("utf-8")
return block.finalize(quarkash=coinbase_amount,
coinbase_address=address)
def add_block(self,
block,
write_db=True,
skip_if_too_old=True,
adjusted_diff: int = None):
""" Add new block.
return True if a longest block is added, False otherwise
There are a couple of optimizations can be done here:
- the root block could only contain minor block header hashes as long as the shards fully validate the headers
- the header (or hashes) are un-ordered as long as they contains valid sub-chains from previous root block
"""
if skip_if_too_old and (
self.tip.height - block.header.height >
self.root_config.MAX_STALE_ROOT_BLOCK_HEIGHT_DIFF):
Logger.info("[R] drop old block {} << {}".format(
block.header.height, self.tip.height))
raise ValueError("block is too old {} << {}".format(
block.header.height, self.tip.height))
start_ms = time_ms()
block_hash, last_minor_block_header_list = self.validate_block(
block, adjusted_diff)
if write_db:
self.db.put_root_block(block, last_minor_block_header_list)
tracking_data_str = block.tracking_data.decode("utf-8")
if tracking_data_str != "":
tracking_data = json.loads(tracking_data_str)
sample = {
"time": time_ms() // 1000,
"shard": "R",
"network": self.env.cluster_config.MONITORING.NETWORK_NAME,
"cluster": self.env.cluster_config.MONITORING.CLUSTER_ID,
"hash": block.header.get_hash().hex(),
"height": block.header.height,
"original_cluster": tracking_data["cluster"],
"inception": tracking_data["inception"],
"creation_latency_ms": tracking_data["creation_ms"],
"add_block_latency_ms": time_ms() - start_ms,
"mined": tracking_data.get("mined", 0),
"propagation_latency_ms":
start_ms - tracking_data.get("mined", 0),
"num_tx": len(block.minor_block_header_list),
}
asyncio.ensure_future(
self.env.cluster_config.kafka_logger.log_kafka_sample_async(
self.env.cluster_config.MONITORING.PROPAGATION_TOPIC,
sample))
if self.tip.total_difficulty < block.header.total_difficulty:
old_tip = self.tip
self.tip = block.header
# TODO: Atomicity during shutdown
self.db.update_tip_hash(block_hash)
self.__rewrite_block_index_to(old_tip, block)
return True
return False
def _post_process_mined_block(block: Union[MinorBlock, RootBlock]):
if isinstance(block, RootBlock):
extra_data = json.loads(block.header.extra_data.decode("utf-8"))
extra_data["mined"] = time_ms()
block.header.extra_data = json.dumps(extra_data).encode("utf-8")
else:
extra_data = json.loads(block.meta.extra_data.decode("utf-8"))
extra_data["mined"] = time_ms()
block.meta.extra_data = json.dumps(extra_data).encode("utf-8")
block.header.hash_meta = block.meta.get_hash()
def post_process_mined_block(self, block: Block):
"""Post-process block to track block propagation latency"""
if isinstance(block, RootBlock):
extra_data = json.loads(block.header.extra_data.decode("utf-8"))
extra_data["mined"] = time_ms()
block.header.extra_data = json.dumps(extra_data).encode("utf-8")
else:
extra_data = json.loads(block.meta.extra_data.decode("utf-8"))
extra_data["mined"] = time_ms()
block.meta.extra_data = json.dumps(extra_data).encode("utf-8")
block.header.hash_meta = block.meta.get_hash()
async def handle_new_block(self, block):
"""
0. if local shard is syncing, doesn't make sense to add, skip
1. if block parent is not in local state/new block pool, discard
2. if already in cache or in local state/new block pool, pass
3. validate: check time, difficulty, POW
4. add it to new minor block broadcast cache
5. broadcast to all peers (minus peer that sent it, optional)
6. add_block() to local state (then remove from cache)
also, broadcast tip if tip is updated (so that peers can sync if they missed blocks, or are new)
"""
if self.synchronizer.running:
# TODO optinal: queue the block if it came from broadcast to so that once sync is over, catch up immediately
return
if block.header.get_hash() in self.state.new_block_pool:
return
if self.state.db.contain_minor_block_by_hash(block.header.get_hash()):
return
if not self.state.db.contain_minor_block_by_hash(
block.header.hash_prev_minor_block):
if block.header.hash_prev_minor_block not in self.state.new_block_pool:
return
# TODO check difficulty and POW here
# one option is to use __validate_block but we may not need the full check
if block.header.create_time > time_ms() // 1000 + 30:
return
self.state.new_block_pool[block.header.get_hash()] = block
self.broadcast_new_block(block)
await self.add_block(block)
def dialin_blacklist(self, remote_address: Address) -> None:
# never blacklist boot nodes
for node in self.whitelist_nodes:
if node.address.ip == remote_address.ip:
return
self._dialin_blacklist[remote_address.ip] = (
time_ms() // 1000 + DIALIN_BLACKLIST_COOLDOWN_SEC)
def simulate_mine(
block,
target_block_time: float,
input_q: MultiProcessingQueue,
output_q: MultiProcessingQueue,
):
"""Sleep until the target time, or a new block is added to queue"""
target_time = block.header.create_time + numpy.random.exponential(
target_block_time)
while True:
time.sleep(0.1)
try:
# raises if queue is empty
block, target_block_time = input_q.get_nowait()
if not block:
output_q.put(None)
return
target_time = block.header.create_time + Miner.__get_block_time(
block, target_block_time)
except Exception:
# got nothing from queue
pass
if time.time() > target_time:
Miner.__log_status(block)
block.header.nonce = random.randint(0, 2**32 - 1)
if isinstance(block, RootBlock):
extra_data = json.loads(
block.header.extra_data.decode("utf-8"))
extra_data["mined"] = time_ms()
# NOTE this actually ruins POW mining; added for perf tracking
block.header.extra_data = json.dumps(extra_data).encode(
"utf-8")
else:
extra_data = json.loads(
block.meta.extra_data.decode("utf-8"))
extra_data["mined"] = time_ms()
# NOTE this actually ruins POW mining; added for perf tracking
block.meta.extra_data = json.dumps(extra_data).encode(
"utf-8")
block.header.hash_meta = block.meta.get_hash()
output_q.put(block)
block, target_block_time = input_q.get(block=True) # blocking
if not block:
output_q.put(None)
return
target_time = block.header.create_time + Miner.__get_block_time(
block, target_block_time)
async def handle_new_block(self, block):
"""
This is a fast path for block propagation. The block is broadcasted to peers before being added to local state.
0. if local shard is syncing, doesn't make sense to add, skip
1. if block parent is not in local state/new block pool, discard (TODO: is this necessary?)
2. if already in cache or in local state/new block pool, pass
3. validate: check time, difficulty, POW
4. add it to new minor block broadcast cache
5. broadcast to all peers (minus peer that sent it, optional)
6. add_block() to local state (then remove from cache)
also, broadcast tip if tip is updated (so that peers can sync if they missed blocks, or are new)
"""
if self.synchronizer.running:
# TODO optional: queue the block if it came from broadcast to so that once sync is over,
# catch up immediately
return
if block.header.get_hash() in self.state.new_block_pool:
return
if self.state.db.contain_minor_block_by_hash(block.header.get_hash()):
return
if not self.state.db.contain_minor_block_by_hash(
block.header.hash_prev_minor_block):
if block.header.hash_prev_minor_block not in self.state.new_block_pool:
return
# Doing full POSW check requires prev block has been added to the state, which could
# slow down block propagation.
# TODO: this is a copy of the code in SyncTask.__validate_block_headers. this it a helper
try:
header = block.header
# Note that PoSW may lower diff, so checks here are necessary but not sufficient
# More checks happen during block addition
shard_config = self.env.quark_chain_config.shards[
header.branch.get_full_shard_id()]
consensus_type = shard_config.CONSENSUS_TYPE
diff = header.difficulty
if shard_config.POSW_CONFIG.ENABLED:
diff //= shard_config.POSW_CONFIG.DIFF_DIVIDER
validate_seal(header, consensus_type, adjusted_diff=diff)
except Exception as e:
Logger.warning(
"[{}] got block with bad seal in handle_new_block: {}".format(
header.branch.to_str(), str(e)))
raise e
if block.header.create_time > time_ms() // 1000 + 30:
return
self.state.new_block_pool[block.header.get_hash()] = block
Logger.info("[{}/{}] got new block with height {}".format(
block.header.branch.get_chain_id(),
block.header.branch.get_shard_id(),
block.header.height,
))
self.broadcast_new_block(block)
await self.add_block(block)
def validate_block_header(self,
block_header: RootBlockHeader,
block_hash=None):
""" Validate the block header.
"""
height = block_header.height
if height < 1:
raise ValueError("unexpected height")
if not self.db.contain_root_block_by_hash(
block_header.hash_prev_block):
raise ValueError("previous hash block mismatch")
prev_block_header = self.db.get_root_block_header_by_hash(
block_header.hash_prev_block)
if prev_block_header.height + 1 != height:
raise ValueError("incorrect block height")
if (block_header.create_time >
time_ms() // 1000 + ALLOWED_FUTURE_BLOCKS_TIME_VALIDATION):
raise ValueError("block too far into future")
if block_header.create_time <= prev_block_header.create_time:
raise ValueError(
"incorrect create time tip time {}, new block time {}".format(
block_header.create_time, prev_block_header.create_time))
if (len(block_header.extra_data) >
self.env.quark_chain_config.BLOCK_EXTRA_DATA_SIZE_LIMIT):
raise ValueError("extra_data in block is too large")
header_hash = block_header.get_hash()
if block_hash is None:
block_hash = header_hash
# Check difficulty, potentially adjusted by guardian mechanism
adjusted_diff = None # type: Optional[int]
if not self.env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK:
diff = self.diff_calc.calculate_diff_with_parent(
prev_block_header, block_header.create_time)
if diff != block_header.difficulty:
raise ValueError("incorrect difficulty")
# lower the difficulty for root block signed by guardian
if block_header.verify_signature(
self.env.quark_chain_config.guardian_public_key):
adjusted_diff = Guardian.adjust_difficulty(
diff, block_header.height)
if (block_header.difficulty + prev_block_header.total_difficulty !=
block_header.total_difficulty):
raise ValueError("incorrect total difficulty")
# Check PoW if applicable
consensus_type = self.root_config.CONSENSUS_TYPE
validate_seal(block_header,
consensus_type,
adjusted_diff=adjusted_diff)
return block_hash
def chk_dialin_blacklist(self, remote_address: Address) -> bool:
if remote_address.ip not in self._dialin_blacklist:
return False
now = time_ms() // 1000
if now >= self._dialin_blacklist[remote_address.ip]:
del self._dialin_blacklist[remote_address.ip]
return False
return True
def add_block(self, block, block_hash=None):
""" Add new block.
return True if a longest block is added, False otherwise
There are a couple of optimizations can be done here:
- the root block could only contain minor block header hashes as long as the shards fully validate the headers
- the header (or hashes) are un-ordered as long as they contains valid sub-chains from previous root block
"""
start_ms = time_ms()
block_hash, last_minor_block_header_list = self.validate_block(
block, block_hash
)
self.db.put_root_block(
block, last_minor_block_header_list, root_block_hash=block_hash
)
tracking_data_str = block.tracking_data.decode("utf-8")
if tracking_data_str != "":
tracking_data = json.loads(tracking_data_str)
sample = {
"time": time_ms() // 1000,
"shard": "R",
"network": self.env.cluster_config.MONITORING.NETWORK_NAME,
"cluster": self.env.cluster_config.MONITORING.CLUSTER_ID,
"hash": block.header.get_hash().hex(),
"height": block.header.height,
"original_cluster": tracking_data["cluster"],
"inception": tracking_data["inception"],
"creation_latency_ms": tracking_data["creation_ms"],
"add_block_latency_ms": time_ms() - start_ms,
"mined": tracking_data.get("mined", 0),
"propagation_latency_ms": start_ms - tracking_data.get("mined", 0),
"num_tx": len(block.minor_block_header_list),
}
asyncio.ensure_future(
self.env.cluster_config.kafka_logger.log_kafka_sample_async(
self.env.cluster_config.MONITORING.PROPAGATION_TOPIC, sample
)
)
if self.tip.height < block.header.height:
self.tip = block.header
self.db.update_tip_hash(block_hash)
self.__rewrite_block_index_to(block)
return True
return False
async def _periodically_unblacklist(self) -> None:
while self.is_operational:
now = time_ms() // 1000
for blk in (self._dialout_blacklist, self._dialin_blacklist):
remove = []
for ip, t in blk.items():
if now >= t:
remove.append(ip)
for ip in remove:
del blk[ip]
await self.sleep(UNBLACKLIST_INTERVAL)
def __validate_block_header(self,
block_header: RootBlockHeader,
adjusted_diff: int = None):
""" Validate the block header.
"""
height = block_header.height
if height < 1:
raise ValueError("unexpected height")
if block_header.version != 0:
raise ValueError("incorrect root block version")
if not self.db.contain_root_block_by_hash(
block_header.hash_prev_block):
raise ValueError("previous hash block mismatch")
prev_block_header = self.db.get_root_block_header_by_hash(
block_header.hash_prev_block)
if prev_block_header.height + 1 != height:
raise ValueError("incorrect block height")
if (block_header.create_time >
time_ms() // 1000 + ALLOWED_FUTURE_BLOCKS_TIME_VALIDATION):
raise ValueError("block too far into future")
if block_header.create_time <= prev_block_header.create_time:
raise ValueError(
"incorrect create time tip time {}, new block time {}".format(
block_header.create_time, prev_block_header.create_time))
if (len(block_header.extra_data) >
self.env.quark_chain_config.BLOCK_EXTRA_DATA_SIZE_LIMIT):
raise ValueError("extra_data in block is too large")
# Check difficulty, potentially adjusted by guardian mechanism
if not self.env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK:
diff = self.diff_calc.calculate_diff_with_parent(
prev_block_header, block_header.create_time)
if diff != block_header.difficulty:
raise ValueError("incorrect difficulty")
if (block_header.difficulty + prev_block_header.total_difficulty !=
block_header.total_difficulty):
raise ValueError("incorrect total difficulty")
# Check PoW if applicable
if not self.env.quark_chain_config.DISABLE_POW_CHECK:
consensus_type = self.root_config.CONSENSUS_TYPE
diff = (adjusted_diff
if adjusted_diff is not None else block_header.difficulty)
validate_seal(block_header, consensus_type, adjusted_diff=diff)
return block_header.get_hash()
async def handle_new_block(self, block):
"""
0. if local shard is syncing, doesn't make sense to add, skip
1. if block parent is not in local state/new block pool, discard
2. if already in cache or in local state/new block pool, pass
3. validate: check time, difficulty, POW
4. add it to new minor block broadcast cache
5. broadcast to all peers (minus peer that sent it, optional)
6. add_block() to local state (then remove from cache)
also, broadcast tip if tip is updated (so that peers can sync if they missed blocks, or are new)
"""
if self.synchronizer.running:
# TODO optinal: queue the block if it came from broadcast to so that once sync is over, catch up immediately
return
if block.header.get_hash() in self.state.new_block_pool:
return
if self.state.db.contain_minor_block_by_hash(block.header.get_hash()):
return
if not self.state.db.contain_minor_block_by_hash(
block.header.hash_prev_minor_block):
if block.header.hash_prev_minor_block not in self.state.new_block_pool:
return
full_shard_id = block.header.branch.get_full_shard_id()
consensus_type = self.env.quark_chain_config.shards[
full_shard_id].CONSENSUS_TYPE
try:
validate_seal(block.header, consensus_type)
except Exception as e:
Logger.warning("[{}] Got block with bad seal: {}".format(
full_shard_id, str(e)))
return
if block.header.create_time > time_ms() // 1000 + 30:
return
self.state.new_block_pool[block.header.get_hash()] = block
Logger.info("[{}/{}] got new block with height {}".format(
block.header.branch.get_chain_id(),
block.header.branch.get_shard_id(),
block.header.height,
))
self.broadcast_new_block(block)
await self.add_block(block)
async def handle_new_block(self, block):
"""
This is a fast path for block propagation. The block is broadcasted to peers before being added to local state.
0. if local shard is syncing, doesn't make sense to add, skip
1. if block parent is not in local state/new block pool, discard (TODO: is this necessary?)
2. if already in cache or in local state/new block pool, pass
3. validate: check time, difficulty, POW
4. add it to new minor block broadcast cache
5. broadcast to all peers (minus peer that sent it, optional)
6. add_block() to local state (then remove from cache)
also, broadcast tip if tip is updated (so that peers can sync if they missed blocks, or are new)
"""
if self.synchronizer.running:
# TODO optional: queue the block if it came from broadcast to so that once sync is over,
# catch up immediately
return
if block.header.get_hash() in self.state.new_block_header_pool:
return
if self.state.db.contain_minor_block_by_hash(block.header.get_hash()):
return
prev_hash, prev_header = block.header.hash_prev_minor_block, None
if prev_hash in self.state.new_block_header_pool:
prev_header = self.state.new_block_header_pool[prev_hash]
else:
prev_header = self.state.db.get_minor_block_header_by_hash(
prev_hash)
if prev_header is None: # Missing prev
return
# Sanity check on timestamp and block height
if (block.header.create_time >
time_ms() // 1000 + ALLOWED_FUTURE_BLOCKS_TIME_BROADCAST):
return
# Ignore old blocks
if (self.state.header_tip
and self.state.header_tip.height - block.header.height >
self.state.shard_config.max_stale_minor_block_height_diff):
return
# There is a race that the root block may not be processed at the moment.
# Ignore it if its root block is not found.
# Otherwise, validate_block() will fail and we will disconnect the peer.
if (self.state.get_root_block_header_by_hash(
block.header.hash_prev_root_block) is None):
return
try:
self.state.validate_block(block)
except Exception as e:
Logger.warning("[{}] got bad block in handle_new_block: {}".format(
block.header.branch.to_str(), str(e)))
raise e
self.state.new_block_header_pool[
block.header.get_hash()] = block.header
Logger.info("[{}/{}] got new block with height {}".format(
block.header.branch.get_chain_id(),
block.header.branch.get_shard_id(),
block.header.height,
))
self.broadcast_new_block(block)
await self.add_block(block)
def _track(block: Block):
"""Post-process block to track block propagation latency"""
tracking_data = json.loads(block.tracking_data.decode("utf-8"))
tracking_data["mined"] = time_ms()
block.tracking_data = json.dumps(tracking_data).encode("utf-8")
