class TransactionRequestHandler(web.RequestHandler):
def initialize(self, cfg: Config, chain: BlockChain):
self.l = DBLogger(self, cfg)
self.chain = chain
def get(self) -> None:
txns = self.chain.transaction_storage.get_all_transactions()
ser_txns = list(map(lambda t: t.serializable(), txns))
resp = {"transactions": ser_txns}
self.set_status(200)
self.write(resp)
def post(self) -> None:
ser = self.request.body.decode('utf-8')
txn = SignedTransaction.deserialize(ser)
if self.chain.transaction_is_valid(txn):
self.l.info("New transaction", txn)
self.chain.add_outstanding_transaction(txn)
self.set_status(200)
self.write(util.generic_ok_response())
else:
self.l.warn("Invalid transaction", txn)
self.set_status(400)
self.write(util.error_response("Invalid transaction"))
class BlockChain(object):
def __init__(self, storage: BlockChainStorage,
transaction_storage: TransactionStorage,
uxto_storage: UXTOStorage, cfg: Config) -> None:
self.storage = storage
self.transaction_storage = transaction_storage
self.uxto_storage = uxto_storage
self.l = DBLogger(self, cfg)
genesis = storage.get_genesis()
if genesis is None:
self.l.info("Storage didn't have genesis. Added.")
storage.add_block(HashedBlock.genesis())
def get_difficulty(self, head: Optional[HashedBlock] = None) -> int:
if head is None:
h = self.get_head()
else:
h = head
if (h.block_num() + 1) < TUNING_SEGMENT_LENGTH:
return difficulty.DEFAULT_DIFFICULTY
elif (h.block_num() + 1) % TUNING_SEGMENT_LENGTH == 0:
diff = self.tuning_segment_difficulty(
h.block.block_config.difficulty, h.block_num())
self.l.info("Head ({}) is at end of segment, retune to {}".format(
h.block_num(), diff))
return diff
else:
return h.block.block_config.difficulty
def get_head(self) -> HashedBlock:
return self.storage.get_head()
def add_block(self, block: HashedBlock) -> None:
if self.storage.has_hash(block.mining_hash()):
self.l.debug("Already have block", block)
return
elif self.block_is_valid(block):
self.l.debug("Store block", block)
self.storage.add_block(block)
for txn in block.block.transactions:
for out in txn.transaction.outputs:
self.l.debug("Add UXTO", txn.txn_hash(), out.output_id)
self.uxto_storage.add_output(txn.txn_hash(), out.to_addr,
out.output_id)
for inp in txn.transaction.inputs:
out = self.get_transaction_output(
inp.output_block_hash, inp.output_transaction_hash,
inp.output_id)
self.l.debug("Claim UXTO", inp.output_transaction_hash,
inp.output_id)
self.uxto_storage.mark_claimed(inp.output_transaction_hash,
inp.output_id)
self._cleanup_outstanding_transactions(block)
self._abandon_blocks()
else:
raise InvalidBlockError("Block is invalid")
def add_outstanding_transaction(self, txn: SignedTransaction) -> None:
if self.transaction_storage.has_transaction(txn.txn_hash()):
self.l.debug("Already have txn", txn)
return
elif self.transaction_is_valid(txn):
self.l.debug("Store transaction", txn)
self.transaction_storage.add_transaction(txn)
else:
raise InvalidTransactionError("Transaction is invalid")
@staticmethod
def genesis_is_valid(block: HashedBlock, l: DBLogger) -> bool:
return block.mining_hash() == HashedBlock.genesis().mining_hash()
def block_is_valid(self, block: HashedBlock) -> bool:
if self.storage.has_hash(block.parent_mining_hash()):
parent = self.storage.get_by_hash(block.parent_mining_hash())
else:
self.l.warn("Parent with hash {} not known".format(
block.parent_mining_hash().hex()))
return False
if self.block_should_be_abandoned(block):
self.l.warn("Block should be abandoned", block)
return False
difficulty = self.get_difficulty(parent)
if block.block.block_config.difficulty != difficulty:
self.l.warn(
"Unexpected difficulty {} for block {} ({}), expected {}".
format(block.block.block_config.difficulty, block.block_num(),
block.mining_hash().hex(), difficulty))
return False
if not block.hash_meets_difficulty():
self.l.warn("Block hash doesn't meet the set difficulty")
return False
if block.block_num() != parent.block_num() + 1:
self.l.warn("Block number isn't parent+1")
return False
n_rewards = 0
for transaction in block.block.transactions:
if not self.transaction_is_valid(transaction):
self.l.warn("Transaction is invalid")
return False
if transaction.is_reward():
n_rewards += 1
if n_rewards != 1:
self.l.warn(
"Invalid number of rewards ({}) in transaction w/ sig{}".
format(n_rewards, transaction.signature))
return False
return True
def transaction_is_valid(self, signed: SignedTransaction) -> bool:
if signed.is_reward():
return self.reward_is_valid(signed)
if not signed.signature_is_valid():
self.l.warn("Transaction signature is invalid (sig {})".format(
signed.signature))
return False
output_sum = Amount(0)
for output in signed.transaction.outputs:
output_sum += output.amount
claimed_prev_outputs: List[TransactionOutput] = []
claimed_sum = Amount(0)
for inp in signed.transaction.inputs:
out = self.get_transaction_output(inp.output_block_hash,
inp.output_transaction_hash,
inp.output_id)
if out is None:
self.l.warn("Output was unknown", out)
return False
if self.uxto_storage.output_is_claimed(inp.output_transaction_hash,
inp.output_id):
self.l.warn("Output already claimed", out)
return False
claimed_sum += out.amount
if out.to_addr != signed.transaction.claimer:
self.l.warn("Output {} can't be claimed by address {}".format(
out, signed.transaction.claimer))
return False
if output_sum != claimed_sum:
self.l.warn("Input/output amount mismatch {} != {}".format(
output_sum, claimed_sum))
return False
return True
def reward_is_valid(self, reward: SignedTransaction) -> bool:
if not reward.signature_is_valid():
self.l.warn("Reward signature is invalid (sig {})".format(
reward.signature))
return False
if len(reward.transaction.outputs) != 1:
self.l.warn("Reward has n_outputs != 1", reward)
return False
if len(reward.transaction.inputs) != 0:
self.l.warn("Reward has inputs", reward)
return False
if reward.transaction.outputs[0].amount != REWARD_AMOUNT:
self.l.warn("Reward has invalid amout", reward)
return False
return True
def get_transaction_output(self, block_hash: Hash, txn_hash: Hash,
output_id: int) -> Optional[TransactionOutput]:
block = self.storage.get_by_hash(block_hash)
if block is None:
self.l.warn(
"Don't have block hash {} when getting txn output".format(
block_hash.hex()))
return None
for signed in block.block.transactions:
if signed.txn_hash() != txn_hash:
continue
for output in signed.transaction.outputs:
if output.output_id == output_id:
return output
self.l.warn("No output with ID", output_id)
return None
def tuning_segment_difficulty(self, current_difficulty: int,
height: int) -> int:
self.l.debug("Calculating tuning segment difficulty using height",
height)
seg_stop = (
(height // TUNING_SEGMENT_LENGTH) + 1) * TUNING_SEGMENT_LENGTH
seg_start = seg_stop - TUNING_SEGMENT_LENGTH
if seg_start == 0:
self.l.debug("Omitting genesis block from tuning calculation")
seg_start = 1
self.l.debug("Getting segment [{}, {})".format(seg_start, seg_stop))
segment = self.storage.get_range(seg_start, seg_stop)
times = map(lambda b: b.mining_timestamp, segment)
adjustment = difficulty_adjustment(times, self.l)
new_difficulty = current_difficulty + adjustment
self.l.debug("Tuning difficulty:", new_difficulty)
if new_difficulty < 0:
self.l.warn(
"Attempted to set new difficulty to {}, clamping to 0".format(
new_difficulty))
new_difficulty = 0
elif new_difficulty > 255:
self.l.warn(
"Attempted to set new difficulty to {}, clamping to 255".
format(new_difficulty))
new_difficulty = 255
return new_difficulty
def block_should_be_abandoned(self, block: HashedBlock) -> bool:
"""
Either the block is in the master chain or it's within 10 blocks of
the current head.
"""
return not (self.block_is_in_master_chain(block) or
(self.get_head().block_num() - block.block_num() <
ABANDONMENT_DEPTH))
def block_is_in_master_chain(self, block: HashedBlock) -> bool:
"""
Use BFS to see if the current head can be reached from the block in
question.
"""
head = self.get_head()
crawl_queue: List[HashedBlock] = [block]
while len(crawl_queue) > 0:
cur = crawl_queue.pop(0)
if cur == head:
return True
else:
crawl_queue.extend(
self.storage.get_by_parent_hash(cur.mining_hash()))
return False
def _cleanup_outstanding_transactions(self, block: HashedBlock) -> None:
self.l.debug("Cleaning up outstanding transactions in block", block)
for txn in block.block.transactions:
if self.transaction_storage.has_transaction(txn.txn_hash()):
self.l.debug("Removing outstanding transaction", txn)
self.transaction_storage.remove_transaction(txn.txn_hash())
else:
self.l.debug("Transaction wasn't oustanding", txn)
def _abandon_blocks(self):
head = self.get_head()
abandon_height = head.block_num() - ABANDONMENT_DEPTH
abandon_candidates = self.storage.get_by_block_num(abandon_height)
for block in abandon_candidates:
if self.block_should_be_abandoned(block):
self.l.debug("Abandon block", block)
self.storage.abandon_block(block)
for txn in block.block.transactions:
if self.transaction_is_valid(txn):
self.l.debug(
"Adding abandoned transaction back to pool", txn)
self.transaction_storage.add_transaction(txn)
else:
self.l.debug("Abandoned transaction no longer valid")
class PeerList(object):
def __init__(self, cfg: Config) -> None:
self.l = DBLogger(self, cfg)
self._conn = sqlite3.connect(cfg.peer_db_path())
self._conn.execute(CREATE_TABLE_SQL)
self._conn.commit()
self.self_peer = Peer(cfg.server_peer_id(),
cfg.server_advertize_addr(),
cfg.server_listen_port())
self.add_peer(self.self_peer)
try:
gateway_id = Peer.request_id(cfg.gateway_address(),
cfg.gateway_port())
self.gateway_peer = Peer(gateway_id, cfg.gateway_address(),
cfg.gateway_port())
self.add_peer(self.gateway_peer)
except requests.exceptions.ConnectionError as e:
self.l.warn("Couldn't connect to gateway", cfg.gateway_address(),
cfg.gateway_port())
self.gateway_peer = None
def add_peer(self, peer: Peer) -> None:
if peer == self.self_peer:
self.l.debug("Not adding self peer", self.self_peer, peer)
return
elif self.has_peer(peer):
self.l.debug("Update peer:", peer)
self._update_peer(peer)
else:
self.l.debug("Insert peer:", peer)
self._ins_peer(peer)
def has_peer(self, peer: Peer) -> bool:
args = {
"peer_id": peer.peer_id,
}
c = self._conn.cursor()
c.execute(HAS_PEER_SQL, args)
return c.fetchone() is not None
def mark_peer_inactive(self, peer: Peer) -> None:
if peer == self.gateway_peer:
self.l.warn("Not marking gateway as inactive")
return
elif not self.has_peer(peer):
self.l.debug("Not marking unknown peer {} inactive".format(peer))
return
args = {
"address": peer.address,
"port": peer.port,
}
self._conn.execute(MARK_PEER_INACTIVE_SQL, args)
self._conn.commit()
def get_all_active_peers(self) -> List[Peer]:
c = self._conn.cursor()
c.execute(GET_ALL_ACTIVE_PEERS_SQL)
return list(map(lambda row: Peer(*row), c))
def random_peer(self) -> Peer:
return random.choice(self.get_all_active_peers())
def peer_sample(self, n: int) -> List[Peer]:
return random.sample(self.get_all_active_peers(), n)
def _update_peer(self, peer: Peer) -> None:
args = {
"last_seen_unix_millis": int(time.time() * 1000),
"active": 1,
"peer_id": peer.peer_id,
"address": peer.address,
"port": peer.port,
}
self._conn.execute(UPDATE_PEER_SQL, args)
self._conn.commit()
def _ins_peer(self, peer: Peer) -> None:
args = {
"last_seen_unix_millis": int(time.time() * 1000),
"active": 1,
"peer_id": peer.peer_id,
"address": peer.address,
"port": peer.port,
}
self._conn.execute(INSERT_PEER_SQL, args)
self._conn.commit()
class ChainClient(object):
def __init__(self, cfg: Config) -> None:
self.l = DBLogger(self, cfg)
self.l.info("Init")
self.peer_list = PeerList(cfg)
self.self_peer = Peer(cfg.server_peer_id(),
cfg.server_advertize_addr(),
cfg.server_listen_port())
self.chain = BlockChain(SqliteBlockChainStorage(cfg),
SqliteTransactionStorage(cfg),
SqliteUXTOStorage(cfg), cfg)
self.cfg = cfg
def poll_forever(self) -> None:
while True:
peers = self.our_peers()
random.shuffle(peers)
peer_sample = peers[:self.cfg.peer_sample_size()]
for peer in peer_sample:
self.l.debug("Syncing with peer", peer)
self.sync(peer)
time.sleep(self.cfg.poll_delay())
def our_peers(self) -> List[Peer]:
peers = self.peer_list.get_all_active_peers()
peers = list(filter(lambda p: p != self.self_peer, peers))
self.l.debug("Sampled peers", peers)
return peers
def sync(self, peer: Peer) -> None:
peers = self.request_peers(peer)
if peers is None:
self.l.debug("Peer not responding", peer)
return
self.l.debug("Peer {} knows about {} peers".format(peer, len(peers)))
for new_peer in peers:
if not self.peer_list.has_peer(
new_peer) and new_peer != self.self_peer:
self.l.info(
"Peer {} previously unknown. Adding.".format(new_peer))
self.peer_list.add_peer(new_peer)
if self.cfg.advertize_self() and not self.self_peer in peers:
self.l.info(
"Peer {} doesn't know about us, telling it.".format(peer))
resp = self._peer_post(peer, "/peer",
{"peers": [self.self_peer.serializable()]})
if resp is None:
self.l.info("Peer didn't respond", peer)
transactions = self.request_transactions(peer)
if transactions is None:
self.l.info("Peer not responding", peer)
return
for txn in transactions:
if not self.chain.transaction_storage.has_transaction(
txn.sha256()):
if self.chain.transaction_is_valid(txn):
self.l.info("New transaction", txn)
self.chain.add_outstanding_transaction(txn)
else:
self.l.warn("Peer sent us an invalid transaction", peer,
txn)
peer_head = self.request_head(peer)
if not peer_head:
self.l.debug("Peer {} didn't give us a head block.".format(peer))
return
if self.chain.storage.has_hash(peer_head.mining_hash()):
self.l.debug("Already have peer's head block")
return
if peer_head.block_num() == 0:
self.l.warn(
"Peer returned a different genesis block than expected", peer,
peer_head)
return
parent = self.request_block(peer_head.parent_mining_hash(), peer)
if parent is None:
self.l.debug("Wat. Peer didn't have parent:",
peer_head.parent_mining_hash())
return
while not self.chain.storage.has_hash(parent.mining_hash()):
parent = self.request_block(parent.parent_mining_hash(), peer)
if parent is None:
self.l.debug("Wat. Peer didn't have parent:",
peer_head.parent_mining_hash())
return
to_request = [parent]
while len(to_request) > 0:
parent = to_request.pop(0)
successors = self.request_successors(parent.mining_hash(), peer)
if successors is None:
self.l.debug("Peer is not responding", peer)
return
for succ in successors: # ( ͡° ͜ʖ ͡°)
self.l.info("New block:", succ)
to_request.append(succ)
self.chain.add_block(succ)
def request_block(self, block_hash: Hash,
peer: Peer) -> Optional[HashedBlock]:
obj = self._peer_get(peer, "/block", {"hex_hash": block_hash.hex()})
if obj is None:
self.l.debug("No HTTP response from peer {}".format(peer))
return None
if obj is None:
self.l.debug("No block from peer", peer)
return None
try:
hb = HashedBlock.from_dict(obj)
except KeyError as e:
self.l.debug("Invalid serialized block from peer {}".format(peer),
exc=e)
return None
return hb
def request_head(self, peer: Peer) -> Optional[HashedBlock]:
head_hash = self.request_head_hash(peer)
if head_hash is None:
self.l.debug("Can't get head block from peer", peer)
return None
obj = self._peer_get(peer, "/block", {"hex_hash": head_hash.hex()})
try:
h = HashedBlock.from_dict(obj)
except KeyError as e:
self.l.debug("Invalid block from peer {}".format(peer), exc=e)
return None
return h
def request_head_hash(self, peer: Peer) -> Optional[Hash]:
obj = self._peer_get(peer, "/chain", {})
if obj is None:
self.l.debug("No head hash from peer", peer)
return None
try:
h = Hash.from_dict(obj["head_hash"])
except KeyError as e:
self.l.debug("No head hash from peer {}".format(peer), exc=e)
return None
return h
def request_successors(self, parent_hash: Hash,
peer: Peer) -> Optional[List[HashedBlock]]:
payload = {"parent_hex_hash": parent_hash.hex()}
obj = self._peer_get(peer, "/blocks", payload)
if obj is None:
self.l.debug("No successors from peer", peer)
return None
if "blocks" not in obj:
self.l.debug("Couldn't turn response into list of blocks", peer,
obj)
return None
new_blocks: List[HashedBlock] = []
for block_obj in obj["blocks"]:
try:
b = HashedBlock.from_dict(block_obj)
except KeyError as e:
self.l.debug("Invalid block object from peer", peer, obj)
return None
new_blocks.append(b)
return new_blocks
def request_peers(self, peer: Peer) -> Optional[List[Peer]]:
obj = self._peer_get(peer, "/peers", {})
if obj is None:
self.l.debug("No peer response from peer", peer)
return None
if not "peers" in obj:
self.l.debug("No peers in response from peer", peer, obj)
return None
new_peers: List[Peer] = []
for peer_obj in obj["peers"]:
try:
new_peer = Peer.from_dict(peer_obj)
except KeyError as e:
self.l.debug("Invalid peer from peer", peer, new_peer)
return []
new_peers.append(new_peer)
return new_peers
def request_transactions(self,
peer: Peer) -> Optional[List[SignedTransaction]]:
obj = self._peer_get(peer, "/outstanding_transactions", {})
if obj is None:
self.l.debug("No peer response from peer", peer)
return None
if not "transactions" in obj:
self.l.debug("No transaction response from peer", peer, obj)
return None
new_transactions: List[SignedTransaction] = []
for txn_obj in obj["transactions"]:
try:
new_txn = SignedTransaction.from_dict(txn_obj)
except KeyError as e:
self.l.debug("Invalid transaction from peer",
peer,
txn_obj,
exc=e)
else:
new_transactions.append(new_txn)
return new_transactions
def _peer_get(self, peer: Peer, path: str,
params: Dict[str, Any]) -> Optional[Dict[str, Any]]:
url = peer.http_url(path)
self.l.debug("get", url, params)
try:
r = requests.get(url, params=params)
resp = r.content
except requests.exceptions.ConnectionError as e:
# todo: mark peer as inactive
self.l.debug("No response from peer", peer, exc=e)
return None
try:
obj = json.loads(resp)
except ValueError as e:
self.l.debug("Invalid JSON response from peer", peer, exc=e)
return None
return obj
def _peer_post(self, peer: Peer, path: str,
payload: Dict[str, Any]) -> Optional[Dict[str, Any]]:
url = peer.http_url(path)
self.l.debug("post", url, payload)
try:
r = requests.post(url, json=payload)
resp = r.content
except requests.exceptions.ConnectionError as e:
# todo: mark peer as inactive
self.l.debug("No response from peer", peer, exc=e)
return None
try:
obj = json.loads(resp)
except ValueError as e:
self.l.debug("Invalid JSON response from peer", peer, exc=e)
return None
return obj
class BlockMiner(object):
def __init__(self,
cfg: Config,
key_pair: Optional[KeyPair] = None) -> None:
self.l = DBLogger(self, cfg)
self.l.info("Init")
self.cfg = cfg
self.client = ChainClient(cfg)
if key_pair is None:
self.key_pair = KeyPair.new()
else:
self.key_pair = key_pair
self.storage = SqliteBlockChainStorage(cfg)
self.transaction_storage = SqliteTransactionStorage(cfg)
self.uxto_storage = SqliteUXTOStorage(cfg)
self.chain = BlockChain(self.storage, self.transaction_storage,
self.uxto_storage, cfg)
def mine_forever(self) -> None:
self.l.info("Miner running")
while True:
head = self.chain.get_head()
self.l.debug("Mining on block {}".format(head.block_num()))
new_block = self.mine_on(head, self.chain.get_difficulty())
if new_block:
self.l.info("Found block {} {}".format(
new_block.block_num(),
new_block.mining_hash().hex()))
self.chain.add_block(new_block)
elif head != self.chain.get_head():
self.l.info("Preempted! Mining on new block", head)
def mine_on(self, parent: HashedBlock,
difficulty: int) -> Optional[HashedBlock]:
reward = self.make_reward()
config = BlockConfig(difficulty)
txns = self.transaction_storage.get_all_transactions()
txns.append(reward)
self.l.debug("Mining on {} txns".format(len(txns)))
block = Block(parent.block_num() + 1, parent.mining_hash(), config,
txns)
hb = HashedBlock(block)
start = time.time()
while time.time() - start < 1.0:
hb.replace_mining_entropy(os.urandom(32))
if hb.hash_meets_difficulty():
return hb
return None
def transactions(self) -> List[SignedTransaction]:
candidate_txns = self.transaction_storage.get_all_transactions()
mine_txns: List[SignedTransaction] = []
for txn in candidate_txns:
if self.chain.transaction_is_valid(txn):
self.l.debug("Txn is is valid for mining", txn)
mine_txns.append(txn)
else:
self.l.warn("Txn is not valid for mining", txn)
mine_txns.append(self.make_reward())
return mine_txns
def make_reward(self) -> SignedTransaction:
reward = Transaction.reward(REWARD_AMOUNT, self.key_pair.address())
return SignedTransaction.sign(reward, self.key_pair)
def make_genesis(self) -> HashedBlock:
b = Block(
0, # block num
None, # parent hash
BlockConfig(DEFAULT_DIFFICULTY),
[])
hb = HashedBlock(b)
while not hb.hash_meets_difficulty():
hb.replace_mining_entropy(os.urandom(32))
return hb