def difficulty_adjustment(block_times: Iterator[Timestamp],
l: DBLogger) -> int:
"""Computes how much the difficulty should be adjusted by (up or down).
Uses the difference of the log of the mean block time delta and the log of
the target delta.
If the mean block time is 2x the target, the adjustment should be -1.
If the mean block time is 4x the target, the adjustment should be -2.
If the mean block time ix 1/8th the target, the adjustment should be +3.
"""
unix_times = list(map(lambda t: t.unix_millis, block_times))
deltas = []
prev = unix_times[0]
for t in unix_times[1:]:
deltas.append(t - prev)
prev = t
mean = sum(deltas) / len(deltas)
l.debug("Segment mean ms:", mean)
l.debug("Target ms:", BLOCK_TIME_TARGET)
log_mean = math.log2(mean)
log_target = math.log2(BLOCK_TIME_TARGET)
log_difference = log_target - log_mean
l.debug("Log difference:", log_difference)
adjustment = int(round(log_target - log_mean))
l.debug("Recommended adjustment:", adjustment)
return adjustment
class BlockChain(object):
def __init__(self, storage: BlockChainStorage,
transaction_storage: TransactionStorage,
uxto_storage: UXTOStorage, cfg: Config) -> None:
self.storage = storage
self.transaction_storage = transaction_storage
self.uxto_storage = uxto_storage
self.l = DBLogger(self, cfg)
genesis = storage.get_genesis()
if genesis is None:
self.l.info("Storage didn't have genesis. Added.")
storage.add_block(HashedBlock.genesis())
def get_difficulty(self, head: Optional[HashedBlock] = None) -> int:
if head is None:
h = self.get_head()
else:
h = head
if (h.block_num() + 1) < TUNING_SEGMENT_LENGTH:
return difficulty.DEFAULT_DIFFICULTY
elif (h.block_num() + 1) % TUNING_SEGMENT_LENGTH == 0:
diff = self.tuning_segment_difficulty(
h.block.block_config.difficulty, h.block_num())
self.l.info("Head ({}) is at end of segment, retune to {}".format(
h.block_num(), diff))
return diff
else:
return h.block.block_config.difficulty
def get_head(self) -> HashedBlock:
return self.storage.get_head()
def add_block(self, block: HashedBlock) -> None:
if self.storage.has_hash(block.mining_hash()):
self.l.debug("Already have block", block)
return
elif self.block_is_valid(block):
self.l.debug("Store block", block)
self.storage.add_block(block)
for txn in block.block.transactions:
for out in txn.transaction.outputs:
self.l.debug("Add UXTO", txn.txn_hash(), out.output_id)
self.uxto_storage.add_output(txn.txn_hash(), out.to_addr,
out.output_id)
for inp in txn.transaction.inputs:
out = self.get_transaction_output(
inp.output_block_hash, inp.output_transaction_hash,
inp.output_id)
self.l.debug("Claim UXTO", inp.output_transaction_hash,
inp.output_id)
self.uxto_storage.mark_claimed(inp.output_transaction_hash,
inp.output_id)
self._cleanup_outstanding_transactions(block)
self._abandon_blocks()
else:
raise InvalidBlockError("Block is invalid")
def add_outstanding_transaction(self, txn: SignedTransaction) -> None:
if self.transaction_storage.has_transaction(txn.txn_hash()):
self.l.debug("Already have txn", txn)
return
elif self.transaction_is_valid(txn):
self.l.debug("Store transaction", txn)
self.transaction_storage.add_transaction(txn)
else:
raise InvalidTransactionError("Transaction is invalid")
@staticmethod
def genesis_is_valid(block: HashedBlock, l: DBLogger) -> bool:
return block.mining_hash() == HashedBlock.genesis().mining_hash()
def block_is_valid(self, block: HashedBlock) -> bool:
if self.storage.has_hash(block.parent_mining_hash()):
parent = self.storage.get_by_hash(block.parent_mining_hash())
else:
self.l.warn("Parent with hash {} not known".format(
block.parent_mining_hash().hex()))
return False
if self.block_should_be_abandoned(block):
self.l.warn("Block should be abandoned", block)
return False
difficulty = self.get_difficulty(parent)
if block.block.block_config.difficulty != difficulty:
self.l.warn(
"Unexpected difficulty {} for block {} ({}), expected {}".
format(block.block.block_config.difficulty, block.block_num(),
block.mining_hash().hex(), difficulty))
return False
if not block.hash_meets_difficulty():
self.l.warn("Block hash doesn't meet the set difficulty")
return False
if block.block_num() != parent.block_num() + 1:
self.l.warn("Block number isn't parent+1")
return False
n_rewards = 0
for transaction in block.block.transactions:
if not self.transaction_is_valid(transaction):
self.l.warn("Transaction is invalid")
return False
if transaction.is_reward():
n_rewards += 1
if n_rewards != 1:
self.l.warn(
"Invalid number of rewards ({}) in transaction w/ sig{}".
format(n_rewards, transaction.signature))
return False
return True
def transaction_is_valid(self, signed: SignedTransaction) -> bool:
if signed.is_reward():
return self.reward_is_valid(signed)
if not signed.signature_is_valid():
self.l.warn("Transaction signature is invalid (sig {})".format(
signed.signature))
return False
output_sum = Amount(0)
for output in signed.transaction.outputs:
output_sum += output.amount
claimed_prev_outputs: List[TransactionOutput] = []
claimed_sum = Amount(0)
for inp in signed.transaction.inputs:
out = self.get_transaction_output(inp.output_block_hash,
inp.output_transaction_hash,
inp.output_id)
if out is None:
self.l.warn("Output was unknown", out)
return False
if self.uxto_storage.output_is_claimed(inp.output_transaction_hash,
inp.output_id):
self.l.warn("Output already claimed", out)
return False
claimed_sum += out.amount
if out.to_addr != signed.transaction.claimer:
self.l.warn("Output {} can't be claimed by address {}".format(
out, signed.transaction.claimer))
return False
if output_sum != claimed_sum:
self.l.warn("Input/output amount mismatch {} != {}".format(
output_sum, claimed_sum))
return False
return True
def reward_is_valid(self, reward: SignedTransaction) -> bool:
if not reward.signature_is_valid():
self.l.warn("Reward signature is invalid (sig {})".format(
reward.signature))
return False
if len(reward.transaction.outputs) != 1:
self.l.warn("Reward has n_outputs != 1", reward)
return False
if len(reward.transaction.inputs) != 0:
self.l.warn("Reward has inputs", reward)
return False
if reward.transaction.outputs[0].amount != REWARD_AMOUNT:
self.l.warn("Reward has invalid amout", reward)
return False
return True
def get_transaction_output(self, block_hash: Hash, txn_hash: Hash,
output_id: int) -> Optional[TransactionOutput]:
block = self.storage.get_by_hash(block_hash)
if block is None:
self.l.warn(
"Don't have block hash {} when getting txn output".format(
block_hash.hex()))
return None
for signed in block.block.transactions:
if signed.txn_hash() != txn_hash:
continue
for output in signed.transaction.outputs:
if output.output_id == output_id:
return output
self.l.warn("No output with ID", output_id)
return None
def tuning_segment_difficulty(self, current_difficulty: int,
height: int) -> int:
self.l.debug("Calculating tuning segment difficulty using height",
height)
seg_stop = (
(height // TUNING_SEGMENT_LENGTH) + 1) * TUNING_SEGMENT_LENGTH
seg_start = seg_stop - TUNING_SEGMENT_LENGTH
if seg_start == 0:
self.l.debug("Omitting genesis block from tuning calculation")
seg_start = 1
self.l.debug("Getting segment [{}, {})".format(seg_start, seg_stop))
segment = self.storage.get_range(seg_start, seg_stop)
times = map(lambda b: b.mining_timestamp, segment)
adjustment = difficulty_adjustment(times, self.l)
new_difficulty = current_difficulty + adjustment
self.l.debug("Tuning difficulty:", new_difficulty)
if new_difficulty < 0:
self.l.warn(
"Attempted to set new difficulty to {}, clamping to 0".format(
new_difficulty))
new_difficulty = 0
elif new_difficulty > 255:
self.l.warn(
"Attempted to set new difficulty to {}, clamping to 255".
format(new_difficulty))
new_difficulty = 255
return new_difficulty
def block_should_be_abandoned(self, block: HashedBlock) -> bool:
"""
Either the block is in the master chain or it's within 10 blocks of
the current head.
"""
return not (self.block_is_in_master_chain(block) or
(self.get_head().block_num() - block.block_num() <
ABANDONMENT_DEPTH))
def block_is_in_master_chain(self, block: HashedBlock) -> bool:
"""
Use BFS to see if the current head can be reached from the block in
question.
"""
head = self.get_head()
crawl_queue: List[HashedBlock] = [block]
while len(crawl_queue) > 0:
cur = crawl_queue.pop(0)
if cur == head:
return True
else:
crawl_queue.extend(
self.storage.get_by_parent_hash(cur.mining_hash()))
return False
def _cleanup_outstanding_transactions(self, block: HashedBlock) -> None:
self.l.debug("Cleaning up outstanding transactions in block", block)
for txn in block.block.transactions:
if self.transaction_storage.has_transaction(txn.txn_hash()):
self.l.debug("Removing outstanding transaction", txn)
self.transaction_storage.remove_transaction(txn.txn_hash())
else:
self.l.debug("Transaction wasn't oustanding", txn)
def _abandon_blocks(self):
head = self.get_head()
abandon_height = head.block_num() - ABANDONMENT_DEPTH
abandon_candidates = self.storage.get_by_block_num(abandon_height)
for block in abandon_candidates:
if self.block_should_be_abandoned(block):
self.l.debug("Abandon block", block)
self.storage.abandon_block(block)
for txn in block.block.transactions:
if self.transaction_is_valid(txn):
self.l.debug(
"Adding abandoned transaction back to pool", txn)
self.transaction_storage.add_transaction(txn)
else:
self.l.debug("Abandoned transaction no longer valid")
class PeerList(object):
def __init__(self, cfg: Config) -> None:
self.l = DBLogger(self, cfg)
self._conn = sqlite3.connect(cfg.peer_db_path())
self._conn.execute(CREATE_TABLE_SQL)
self._conn.commit()
self.self_peer = Peer(cfg.server_peer_id(),
cfg.server_advertize_addr(),
cfg.server_listen_port())
self.add_peer(self.self_peer)
try:
gateway_id = Peer.request_id(cfg.gateway_address(),
cfg.gateway_port())
self.gateway_peer = Peer(gateway_id, cfg.gateway_address(),
cfg.gateway_port())
self.add_peer(self.gateway_peer)
except requests.exceptions.ConnectionError as e:
self.l.warn("Couldn't connect to gateway", cfg.gateway_address(),
cfg.gateway_port())
self.gateway_peer = None
def add_peer(self, peer: Peer) -> None:
if peer == self.self_peer:
self.l.debug("Not adding self peer", self.self_peer, peer)
return
elif self.has_peer(peer):
self.l.debug("Update peer:", peer)
self._update_peer(peer)
else:
self.l.debug("Insert peer:", peer)
self._ins_peer(peer)
def has_peer(self, peer: Peer) -> bool:
args = {
"peer_id": peer.peer_id,
}
c = self._conn.cursor()
c.execute(HAS_PEER_SQL, args)
return c.fetchone() is not None
def mark_peer_inactive(self, peer: Peer) -> None:
if peer == self.gateway_peer:
self.l.warn("Not marking gateway as inactive")
return
elif not self.has_peer(peer):
self.l.debug("Not marking unknown peer {} inactive".format(peer))
return
args = {
"address": peer.address,
"port": peer.port,
}
self._conn.execute(MARK_PEER_INACTIVE_SQL, args)
self._conn.commit()
def get_all_active_peers(self) -> List[Peer]:
c = self._conn.cursor()
c.execute(GET_ALL_ACTIVE_PEERS_SQL)
return list(map(lambda row: Peer(*row), c))
def random_peer(self) -> Peer:
return random.choice(self.get_all_active_peers())
def peer_sample(self, n: int) -> List[Peer]:
return random.sample(self.get_all_active_peers(), n)
def _update_peer(self, peer: Peer) -> None:
args = {
"last_seen_unix_millis": int(time.time() * 1000),
"active": 1,
"peer_id": peer.peer_id,
"address": peer.address,
"port": peer.port,
}
self._conn.execute(UPDATE_PEER_SQL, args)
self._conn.commit()
def _ins_peer(self, peer: Peer) -> None:
args = {
"last_seen_unix_millis": int(time.time() * 1000),
"active": 1,
"peer_id": peer.peer_id,
"address": peer.address,
"port": peer.port,
}
self._conn.execute(INSERT_PEER_SQL, args)
self._conn.commit()
class ChainClient(object):
def __init__(self, cfg: Config) -> None:
self.l = DBLogger(self, cfg)
self.l.info("Init")
self.peer_list = PeerList(cfg)
self.self_peer = Peer(cfg.server_peer_id(),
cfg.server_advertize_addr(),
cfg.server_listen_port())
self.chain = BlockChain(SqliteBlockChainStorage(cfg),
SqliteTransactionStorage(cfg),
SqliteUXTOStorage(cfg), cfg)
self.cfg = cfg
def poll_forever(self) -> None:
while True:
peers = self.our_peers()
random.shuffle(peers)
peer_sample = peers[:self.cfg.peer_sample_size()]
for peer in peer_sample:
self.l.debug("Syncing with peer", peer)
self.sync(peer)
time.sleep(self.cfg.poll_delay())
def our_peers(self) -> List[Peer]:
peers = self.peer_list.get_all_active_peers()
peers = list(filter(lambda p: p != self.self_peer, peers))
self.l.debug("Sampled peers", peers)
return peers
def sync(self, peer: Peer) -> None:
peers = self.request_peers(peer)
if peers is None:
self.l.debug("Peer not responding", peer)
return
self.l.debug("Peer {} knows about {} peers".format(peer, len(peers)))
for new_peer in peers:
if not self.peer_list.has_peer(
new_peer) and new_peer != self.self_peer:
self.l.info(
"Peer {} previously unknown. Adding.".format(new_peer))
self.peer_list.add_peer(new_peer)
if self.cfg.advertize_self() and not self.self_peer in peers:
self.l.info(
"Peer {} doesn't know about us, telling it.".format(peer))
resp = self._peer_post(peer, "/peer",
{"peers": [self.self_peer.serializable()]})
if resp is None:
self.l.info("Peer didn't respond", peer)
transactions = self.request_transactions(peer)
if transactions is None:
self.l.info("Peer not responding", peer)
return
for txn in transactions:
if not self.chain.transaction_storage.has_transaction(
txn.sha256()):
if self.chain.transaction_is_valid(txn):
self.l.info("New transaction", txn)
self.chain.add_outstanding_transaction(txn)
else:
self.l.warn("Peer sent us an invalid transaction", peer,
txn)
peer_head = self.request_head(peer)
if not peer_head:
self.l.debug("Peer {} didn't give us a head block.".format(peer))
return
if self.chain.storage.has_hash(peer_head.mining_hash()):
self.l.debug("Already have peer's head block")
return
if peer_head.block_num() == 0:
self.l.warn(
"Peer returned a different genesis block than expected", peer,
peer_head)
return
parent = self.request_block(peer_head.parent_mining_hash(), peer)
if parent is None:
self.l.debug("Wat. Peer didn't have parent:",
peer_head.parent_mining_hash())
return
while not self.chain.storage.has_hash(parent.mining_hash()):
parent = self.request_block(parent.parent_mining_hash(), peer)
if parent is None:
self.l.debug("Wat. Peer didn't have parent:",
peer_head.parent_mining_hash())
return
to_request = [parent]
while len(to_request) > 0:
parent = to_request.pop(0)
successors = self.request_successors(parent.mining_hash(), peer)
if successors is None:
self.l.debug("Peer is not responding", peer)
return
for succ in successors: # ( ͡° ͜ʖ ͡°)
self.l.info("New block:", succ)
to_request.append(succ)
self.chain.add_block(succ)
def request_block(self, block_hash: Hash,
peer: Peer) -> Optional[HashedBlock]:
obj = self._peer_get(peer, "/block", {"hex_hash": block_hash.hex()})
if obj is None:
self.l.debug("No HTTP response from peer {}".format(peer))
return None
if obj is None:
self.l.debug("No block from peer", peer)
return None
try:
hb = HashedBlock.from_dict(obj)
except KeyError as e:
self.l.debug("Invalid serialized block from peer {}".format(peer),
exc=e)
return None
return hb
def request_head(self, peer: Peer) -> Optional[HashedBlock]:
head_hash = self.request_head_hash(peer)
if head_hash is None:
self.l.debug("Can't get head block from peer", peer)
return None
obj = self._peer_get(peer, "/block", {"hex_hash": head_hash.hex()})
try:
h = HashedBlock.from_dict(obj)
except KeyError as e:
self.l.debug("Invalid block from peer {}".format(peer), exc=e)
return None
return h
def request_head_hash(self, peer: Peer) -> Optional[Hash]:
obj = self._peer_get(peer, "/chain", {})
if obj is None:
self.l.debug("No head hash from peer", peer)
return None
try:
h = Hash.from_dict(obj["head_hash"])
except KeyError as e:
self.l.debug("No head hash from peer {}".format(peer), exc=e)
return None
return h
def request_successors(self, parent_hash: Hash,
peer: Peer) -> Optional[List[HashedBlock]]:
payload = {"parent_hex_hash": parent_hash.hex()}
obj = self._peer_get(peer, "/blocks", payload)
if obj is None:
self.l.debug("No successors from peer", peer)
return None
if "blocks" not in obj:
self.l.debug("Couldn't turn response into list of blocks", peer,
obj)
return None
new_blocks: List[HashedBlock] = []
for block_obj in obj["blocks"]:
try:
b = HashedBlock.from_dict(block_obj)
except KeyError as e:
self.l.debug("Invalid block object from peer", peer, obj)
return None
new_blocks.append(b)
return new_blocks
def request_peers(self, peer: Peer) -> Optional[List[Peer]]:
obj = self._peer_get(peer, "/peers", {})
if obj is None:
self.l.debug("No peer response from peer", peer)
return None
if not "peers" in obj:
self.l.debug("No peers in response from peer", peer, obj)
return None
new_peers: List[Peer] = []
for peer_obj in obj["peers"]:
try:
new_peer = Peer.from_dict(peer_obj)
except KeyError as e:
self.l.debug("Invalid peer from peer", peer, new_peer)
return []
new_peers.append(new_peer)
return new_peers
def request_transactions(self,
peer: Peer) -> Optional[List[SignedTransaction]]:
obj = self._peer_get(peer, "/outstanding_transactions", {})
if obj is None:
self.l.debug("No peer response from peer", peer)
return None
if not "transactions" in obj:
self.l.debug("No transaction response from peer", peer, obj)
return None
new_transactions: List[SignedTransaction] = []
for txn_obj in obj["transactions"]:
try:
new_txn = SignedTransaction.from_dict(txn_obj)
except KeyError as e:
self.l.debug("Invalid transaction from peer",
peer,
txn_obj,
exc=e)
else:
new_transactions.append(new_txn)
return new_transactions
def _peer_get(self, peer: Peer, path: str,
params: Dict[str, Any]) -> Optional[Dict[str, Any]]:
url = peer.http_url(path)
self.l.debug("get", url, params)
try:
r = requests.get(url, params=params)
resp = r.content
except requests.exceptions.ConnectionError as e:
# todo: mark peer as inactive
self.l.debug("No response from peer", peer, exc=e)
return None
try:
obj = json.loads(resp)
except ValueError as e:
self.l.debug("Invalid JSON response from peer", peer, exc=e)
return None
return obj
def _peer_post(self, peer: Peer, path: str,
payload: Dict[str, Any]) -> Optional[Dict[str, Any]]:
url = peer.http_url(path)
self.l.debug("post", url, payload)
try:
r = requests.post(url, json=payload)
resp = r.content
except requests.exceptions.ConnectionError as e:
# todo: mark peer as inactive
self.l.debug("No response from peer", peer, exc=e)
return None
try:
obj = json.loads(resp)
except ValueError as e:
self.l.debug("Invalid JSON response from peer", peer, exc=e)
return None
return obj
class BlockMiner(object):
def __init__(self,
cfg: Config,
key_pair: Optional[KeyPair] = None) -> None:
self.l = DBLogger(self, cfg)
self.l.info("Init")
self.cfg = cfg
self.client = ChainClient(cfg)
if key_pair is None:
self.key_pair = KeyPair.new()
else:
self.key_pair = key_pair
self.storage = SqliteBlockChainStorage(cfg)
self.transaction_storage = SqliteTransactionStorage(cfg)
self.uxto_storage = SqliteUXTOStorage(cfg)
self.chain = BlockChain(self.storage, self.transaction_storage,
self.uxto_storage, cfg)
def mine_forever(self) -> None:
self.l.info("Miner running")
while True:
head = self.chain.get_head()
self.l.debug("Mining on block {}".format(head.block_num()))
new_block = self.mine_on(head, self.chain.get_difficulty())
if new_block:
self.l.info("Found block {} {}".format(
new_block.block_num(),
new_block.mining_hash().hex()))
self.chain.add_block(new_block)
elif head != self.chain.get_head():
self.l.info("Preempted! Mining on new block", head)
def mine_on(self, parent: HashedBlock,
difficulty: int) -> Optional[HashedBlock]:
reward = self.make_reward()
config = BlockConfig(difficulty)
txns = self.transaction_storage.get_all_transactions()
txns.append(reward)
self.l.debug("Mining on {} txns".format(len(txns)))
block = Block(parent.block_num() + 1, parent.mining_hash(), config,
txns)
hb = HashedBlock(block)
start = time.time()
while time.time() - start < 1.0:
hb.replace_mining_entropy(os.urandom(32))
if hb.hash_meets_difficulty():
return hb
return None
def transactions(self) -> List[SignedTransaction]:
candidate_txns = self.transaction_storage.get_all_transactions()
mine_txns: List[SignedTransaction] = []
for txn in candidate_txns:
if self.chain.transaction_is_valid(txn):
self.l.debug("Txn is is valid for mining", txn)
mine_txns.append(txn)
else:
self.l.warn("Txn is not valid for mining", txn)
mine_txns.append(self.make_reward())
return mine_txns
def make_reward(self) -> SignedTransaction:
reward = Transaction.reward(REWARD_AMOUNT, self.key_pair.address())
return SignedTransaction.sign(reward, self.key_pair)
def make_genesis(self) -> HashedBlock:
b = Block(
0, # block num
None, # parent hash
BlockConfig(DEFAULT_DIFFICULTY),
[])
hb = HashedBlock(b)
while not hb.hash_meets_difficulty():
hb.replace_mining_entropy(os.urandom(32))
return hb