def mine_next_block(parent, coinbase=None, transactions=[]): if coinbase: c = Chain(env=parent.env, genesis=parent, coinbase=coinbase) else: c = Chain(env=parent.env, genesis=parent) for tx in transactions: c.add_transaction(tx) block = mine_on_chain(c) return block
def mine_next_block(parent, coinbase=None, transactions=[]): if coinbase: c = Chain(env=parent.env, genesis=parent, coinbase=coinbase) else: c = Chain(env=parent.env, genesis=parent) for tx in transactions: c.add_transaction(tx) block = mine_on_chain(c) return block
class ChainService(WiredService): """ Manages the chain and requests to it. """ # required by BaseService name = 'chain' default_config = dict(eth=dict(privkey_hex='')) # required by WiredService wire_protocol = eth_protocol.ETHProtocol # create for each peer # initialized after configure: chain = None genesis = None synchronizer = None config = None block_queue_size = 1024 transaction_queue_size = 1024 def __init__(self, app): self.config = app.config self.db = app.services.db assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') self.chain = Chain(self.db, new_head_cb=self._on_new_head) self.synchronizer = Synchronizer(self, force_sync=None) self.chain.coinbase = privtoaddr(self.config['eth']['privkey_hex'].decode('hex')) self.block_queue = Queue(maxsize=self.block_queue_size) self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.broadcast_filter = DuplicatesFilter() def _on_new_head(self, block): pass def add_block(self, t_block, proto): "adds a block to the block_queue and spawns _add_block if not running" self.block_queue.put((t_block, proto)) # blocks if full if not self.add_blocks_lock: self.add_blocks_lock = True gevent.spawn(self._add_blocks) def _add_blocks(self): log.debug('add_blocks', qsize=self.block_queue.qsize()) try: while not self.block_queue.empty(): t_block, proto = self.block_queue.get() if t_block.header.hash in self.chain: log.warn('known block', block=t_block) continue if t_block.header.prevhash not in self.chain: log.warn('missing parent', block=t_block) continue if not t_block.header.check_pow(): log.warn('invalid pow', block=t_block) # FIXME ban node continue try: # deserialize st = time.time() block = t_block.to_block(db=self.chain.db) elapsed = time.time() - st log.debug('deserialized', elapsed='%.2fs' % elapsed, gas_used=block.gas_used, gpsec=int(block.gas_used / elapsed)) except processblock.InvalidTransaction as e: log.warn('invalid transaction', block=t_block, error=e) # FIXME ban node continue if self.chain.add_block(block): log.debug('added', block=block) gevent.sleep(0.001) finally: self.add_blocks_lock = False def broadcast_newblock(self, block, chain_difficulty, origin=None): assert isinstance(block, eth_protocol.TransientBlock) if self.broadcast_filter.known(block.header.hash): log.debug('already broadcasted block') else: log.debug('broadcasting newblock', origin=origin) bcast = self.app.services.peermanager.broadcast bcast(eth_protocol.ETHProtocol, 'newblock', args=(block, chain_difficulty), num_peers=None, exclude_protos=[origin]) # wire protocol receivers ########### def on_wire_protocol_start(self, proto): log.debug('on_wire_protocol_start', proto=proto) assert isinstance(proto, self.wire_protocol) # register callbacks proto.receive_status_callbacks.append(self.on_receive_status) proto.receive_transactions_callbacks.append(self.on_receive_transactions) proto.receive_getblockhashes_callbacks.append(self.on_receive_getblockhashes) proto.receive_blockhashes_callbacks.append(self.on_receive_blockhashes) proto.receive_getblocks_callbacks.append(self.on_receive_getblocks) proto.receive_blocks_callbacks.append(self.on_receive_blocks) proto.receive_newblock_callbacks.append(self.on_receive_newblock) # send status head = self.chain.head proto.send_status(chain_difficulty=head.chain_difficulty(), chain_head_hash=head.hash, genesis_hash=self.chain.genesis.hash) def on_wire_protocol_stop(self, proto): assert isinstance(proto, self.wire_protocol) log.debug('on_wire_protocol_stop', proto=proto) def on_receive_status(self, proto, eth_version, network_id, chain_difficulty, chain_head_hash, genesis_hash): log.debug('status received', proto=proto, eth_version=eth_version) assert eth_version == proto.version, (eth_version, proto.version) if network_id != proto.network_id: log.warn("invalid network id", remote_id=proto.network_id, network_id=network_id) raise eth_protocol.ETHProtocolError('wrong network_id') # check genesis if genesis_hash != self.chain.genesis.hash: log.warn("invalid genesis hash", remote_id=proto, genesis=genesis_hash.encode('hex')) raise eth_protocol.ETHProtocolError('wrong genesis block') # request chain self.synchronizer.receive_status(proto, chain_head_hash, chain_difficulty) # send transactions transactions = self.chain.get_transactions() if transactions: log.debug("sending transactions", remote_id=proto) proto.send_transactions(*transactions) # transactions def on_receive_transactions(self, proto, transactions): "receives rlp.decoded serialized" log.debug('remote_transactions_received', count=len(transactions), remote_id=proto) log.debug('skipping, FIXME') return for tx in transactions: # fixme bloomfilter self.chain.add_transaction(tx) # blockhashes ########### def on_receive_getblockhashes(self, proto, child_block_hash, count): log.debug("handle_get_blockhashes", count=count, block_hash=encode_hex(child_block_hash)) max_hashes = min(count, self.wire_protocol.max_getblockhashes_count) found = [] if child_block_hash not in self.chain: log.debug("unknown block") proto.send_blockhashes(*[]) return last = child_block_hash while len(found) < max_hashes: last = rlp.decode_lazy(self.chain.db.get(last))[0][0] if last: found.append(last) else: break log.debug("sending: found block_hashes", count=len(found)) proto.send_blockhashes(*found) def on_receive_blockhashes(self, proto, blockhashes): if blockhashes: log.debug("on_receive_blockhashes", count=len(blockhashes), remote_id=proto, first=encode_hex(blockhashes[0]), last=encode_hex(blockhashes[-1])) else: log.debug("recv 0 remote block hashes, signifying genesis block") self.synchronizer.receive_blockhashes(proto, blockhashes) # blocks ################ def on_receive_getblocks(self, proto, blockhashes): log.debug("on_receive_getblocks", count=len(blockhashes)) found = [] for bh in blockhashes[:self.wire_protocol.max_getblocks_count]: try: found.append(self.chain.db.get(bh)) except KeyError: log.debug("unknown block requested", block_hash=encode_hex(bh)) if found: log.debug("found", count=len(found)) proto.send_blocks(*found) def on_receive_blocks(self, proto, transient_blocks): log.debug("recv blocks", count=len(transient_blocks), remote_id=proto, highest_number=max(x.header.number for x in transient_blocks)) if transient_blocks: self.synchronizer.receive_blocks(proto, transient_blocks) def on_receive_newblock(self, proto, block, chain_difficulty): log.debug("recv newblock", block=block, remote_id=proto) self.synchronizer.receive_newblock(proto, block, chain_difficulty)
class ChainService(WiredService): """ Manages the chain and requests to it. """ # required by BaseService name = 'chain' default_config = dict(eth=dict(network_id=0, genesis_nonce=GENESIS_NONCE.encode('hex'))) # required by WiredService wire_protocol = eth_protocol.ETHProtocol # create for each peer # initialized after configure: chain = None genesis = None synchronizer = None config = None block_queue_size = 1024 transaction_queue_size = 1024 processed_gas = 0 processed_elapsed = 0 def __init__(self, app): self.config = app.config self.db = app.services.db assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') coinbase = app.services.accounts.coinbase _genesis = genesis(self.db, nonce=self.config['eth']['genesis_nonce'].decode('hex')) self.chain = Chain(self.db, genesis=_genesis, new_head_cb=self._on_new_head, coinbase=coinbase) log.info('chain at', number=self.chain.head.number) self.synchronizer = Synchronizer(self, force_sync=None) self.block_queue = Queue(maxsize=self.block_queue_size) self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.add_transaction_lock = gevent.lock.Semaphore() self.broadcast_filter = DuplicatesFilter() self.on_new_head_cbs = [] self.on_new_head_candidate_cbs = [] self.newblock_processing_times = deque(maxlen=1000) # gevent.spawn(update_watcher, self) @property def is_syncing(self): return self.synchronizer.synctask is not None @property def is_mining(self): if 'pow' in self.app.services: return self.app.services.pow.active return False def _on_new_head(self, block): for cb in self.on_new_head_cbs: cb(block) self._on_new_head_candidate() # we implicitly have a new head_candidate def _on_new_head_candidate(self): for cb in self.on_new_head_candidate_cbs: cb(self.chain.head_candidate) def add_transaction(self, tx, origin=None): log.debug('add_transaction', locked=self.add_transaction_lock.locked(), tx=tx) assert isinstance(tx, Transaction) assert origin is None or isinstance(origin, BaseProtocol) if tx.hash in self.broadcast_filter: log.debug('discarding known tx') # discard early return # validate transaction try: validate_transaction(self.chain.head_candidate, tx) log.debug('valid tx, broadcasting') self.broadcast_transaction(tx, origin=origin) # asap except InvalidTransaction as e: log.debug('invalid tx', error=e) return if origin is not None: # not locally added via jsonrpc if not self.is_mining or self.is_syncing: log.debug('discarding tx', syncing=self.is_syncing, mining=self.is_mining) return self.add_transaction_lock.acquire() success = self.chain.add_transaction(tx) self.add_transaction_lock.release() if success: self._on_new_head_candidate() def add_block(self, t_block, proto): "adds a block to the block_queue and spawns _add_block if not running" self.block_queue.put((t_block, proto)) # blocks if full if not self.add_blocks_lock: self.add_blocks_lock = True # need to lock here (ctx switch is later) gevent.spawn(self._add_blocks) def add_mined_block(self, block): log.debug('adding mined block', block=block) assert isinstance(block, Block) assert block.header.check_pow() if self.chain.add_block(block): log.debug('added', block=block, ts=time.time()) assert block == self.chain.head self.broadcast_newblock(block, chain_difficulty=block.chain_difficulty()) def knows_block(self, block_hash): "if block is in chain or in queue" if block_hash in self.chain: return True # check if queued or processed for i in range(len(self.block_queue.queue)): if block_hash == self.block_queue.queue[i][0].header.hash: return True return False def _add_blocks(self): log.debug('add_blocks', qsize=self.block_queue.qsize(), add_tx_lock=self.add_transaction_lock.locked()) assert self.add_blocks_lock is True self.add_transaction_lock.acquire() try: while not self.block_queue.empty(): t_block, proto = self.block_queue.peek() # peek: knows_block while processing if t_block.header.hash in self.chain: log.warn('known block', block=t_block) self.block_queue.get() continue if t_block.header.prevhash not in self.chain: log.warn('missing parent', block=t_block) self.block_queue.get() continue # FIXME, this is also done in validation and in synchronizer for new_blocks if not t_block.header.check_pow(): log.warn('invalid pow', block=t_block, FIXME='ban node') self.block_queue.get() continue try: # deserialize st = time.time() block = t_block.to_block(db=self.chain.db) elapsed = time.time() - st log.debug('deserialized', elapsed='%.4fs' % elapsed, ts=time.time(), gas_used=block.gas_used, gpsec=self.gpsec(block.gas_used, elapsed)) except processblock.InvalidTransaction as e: log.warn('invalid transaction', block=t_block, error=e, FIXME='ban node') self.block_queue.get() continue except VerificationFailed as e: log.warn('verification failed', error=e, FIXME='ban node') self.block_queue.get() continue log.debug('adding', block=block, ts=time.time()) if self.chain.add_block(block, forward_pending_transactions=self.is_mining): now = time.time() log.debug('added', block=block, ts=now, txs=len(block.get_transactions())) if t_block.newblock_timestamp: total = now - t_block.newblock_timestamp self.newblock_processing_times.append(total) avg = statistics.mean(self.newblock_processing_times) med = statistics.median(self.newblock_processing_times) max_ = max(self.newblock_processing_times) min_ = min(self.newblock_processing_times) log.debug('processing time', last=total, avg=avg, max=max_, min=min_, median=med) else: log.warn('could not add', block=block) self.block_queue.get() # remove block from queue (we peeked only) gevent.sleep(0.001) finally: self.add_blocks_lock = False self.add_transaction_lock.release() def gpsec(self, gas_spent=0, elapsed=0): if gas_spent: self.processed_gas += gas_spent self.processed_elapsed += elapsed return int(self.processed_gas / (0.001 + self.processed_elapsed)) def broadcast_newblock(self, block, chain_difficulty=None, origin=None): if not chain_difficulty: assert block.hash in self.chain chain_difficulty = block.chain_difficulty() assert isinstance(block, (eth_protocol.TransientBlock, Block)) if self.broadcast_filter.update(block.header.hash): log.debug('broadcasting newblock', origin=origin) bcast = self.app.services.peermanager.broadcast bcast(eth_protocol.ETHProtocol, 'newblock', args=(block, chain_difficulty), exclude_peers=[origin.peer] if origin else []) else: log.debug('already broadcasted block') def broadcast_transaction(self, tx, origin=None): assert isinstance(tx, Transaction) if self.broadcast_filter.update(tx.hash): log.debug('broadcasting tx', origin=origin) bcast = self.app.services.peermanager.broadcast bcast(eth_protocol.ETHProtocol, 'transactions', args=(tx,), exclude_peers=[origin.peer] if origin else []) else: log.debug('already broadcasted tx') # wire protocol receivers ########### def on_wire_protocol_start(self, proto): log.debug('----------------------------------') log.debug('on_wire_protocol_start', proto=proto) assert isinstance(proto, self.wire_protocol) # register callbacks proto.receive_status_callbacks.append(self.on_receive_status) proto.receive_transactions_callbacks.append(self.on_receive_transactions) proto.receive_getblockhashes_callbacks.append(self.on_receive_getblockhashes) proto.receive_blockhashes_callbacks.append(self.on_receive_blockhashes) proto.receive_getblocks_callbacks.append(self.on_receive_getblocks) proto.receive_blocks_callbacks.append(self.on_receive_blocks) proto.receive_newblock_callbacks.append(self.on_receive_newblock) proto.receive_newblockhashes_callbacks.append(self.on_newblockhashes) # send status head = self.chain.head proto.send_status(chain_difficulty=head.chain_difficulty(), chain_head_hash=head.hash, genesis_hash=self.chain.genesis.hash) def on_wire_protocol_stop(self, proto): assert isinstance(proto, self.wire_protocol) log.debug('----------------------------------') log.debug('on_wire_protocol_stop', proto=proto) def on_receive_status(self, proto, eth_version, network_id, chain_difficulty, chain_head_hash, genesis_hash): log.debug('----------------------------------') log.debug('status received', proto=proto, eth_version=eth_version) assert eth_version == proto.version, (eth_version, proto.version) if network_id != self.config['eth'].get('network_id', proto.network_id): log.warn("invalid network id", remote_network_id=network_id, expected_network_id=self.config['eth'].get('network_id', proto.network_id)) raise eth_protocol.ETHProtocolError('wrong network_id') # check genesis if genesis_hash != self.chain.genesis.hash: log.warn("invalid genesis hash", remote_id=proto, genesis=genesis_hash.encode('hex')) raise eth_protocol.ETHProtocolError('wrong genesis block') # request chain self.synchronizer.receive_status(proto, chain_head_hash, chain_difficulty) # send transactions transactions = self.chain.get_transactions() if transactions: log.debug("sending transactions", remote_id=proto) proto.send_transactions(*transactions) # transactions def on_receive_transactions(self, proto, transactions): "receives rlp.decoded serialized" log.debug('----------------------------------') log.debug('remote_transactions_received', count=len(transactions), remote_id=proto) for tx in transactions: self.add_transaction(tx, origin=proto) # blockhashes ########### def on_newblockhashes(self, proto, newblockhashes): """ msg sent out if not the full block is propagated chances are high, that we get the newblock, though. """ log.debug('----------------------------------') log.debug("recv newnewblockhashes", num=len(newblockhashes), remote_id=proto) self.synchronizer.receive_newblockhashes(proto, newblockhashes) def on_receive_getblockhashes(self, proto, child_block_hash, count): log.debug('----------------------------------') log.debug("handle_get_blockhashes", count=count, block_hash=encode_hex(child_block_hash)) max_hashes = min(count, self.wire_protocol.max_getblockhashes_count) found = [] if child_block_hash not in self.chain: log.debug("unknown block") proto.send_blockhashes(*[]) return last = child_block_hash while len(found) < max_hashes: try: last = rlp.decode_lazy(self.chain.db.get(last))[0][0] # [head][prevhash] except KeyError: # this can happen if we started a chain download, which did not complete # should not happen if the hash is part of the canonical chain log.warn('KeyError in getblockhashes', hash=last) break if last: found.append(last) else: break log.debug("sending: found block_hashes", count=len(found)) proto.send_blockhashes(*found) def on_receive_blockhashes(self, proto, blockhashes): log.debug('----------------------------------') if blockhashes: log.debug("on_receive_blockhashes", count=len(blockhashes), remote_id=proto, first=encode_hex(blockhashes[0]), last=encode_hex(blockhashes[-1])) else: log.debug("recv 0 remote block hashes, signifying genesis block") self.synchronizer.receive_blockhashes(proto, blockhashes) # blocks ################ def on_receive_getblocks(self, proto, blockhashes): log.debug('----------------------------------') log.debug("on_receive_getblocks", count=len(blockhashes)) found = [] for bh in blockhashes[:self.wire_protocol.max_getblocks_count]: try: found.append(self.chain.db.get(bh)) except KeyError: log.debug("unknown block requested", block_hash=encode_hex(bh)) if found: log.debug("found", count=len(found)) proto.send_blocks(*found) def on_receive_blocks(self, proto, transient_blocks): log.debug('----------------------------------') blk_number = max(x.header.number for x in transient_blocks) if transient_blocks else 0 log.debug("recv blocks", count=len(transient_blocks), remote_id=proto, highest_number=blk_number) if transient_blocks: self.synchronizer.receive_blocks(proto, transient_blocks) def on_receive_newblock(self, proto, block, chain_difficulty): log.debug('----------------------------------') log.debug("recv newblock", block=block, remote_id=proto) self.synchronizer.receive_newblock(proto, block, chain_difficulty)
class Validator(): def __init__(self, genesis, key, network, env, time_offset=5): # Create a chain object self.chain = Chain(genesis, env=env) # Use the validator's time as the chain's time self.chain.time = lambda: self.get_timestamp() # My private key self.key = key # My address self.address = privtoaddr(key) # My randao self.randao = RandaoManager(sha3(self.key)) # Pointer to the test p2p network self.network = network # Record of objects already received and processed self.received_objects = {} # The minimum eligible timestamp given a particular number of skips self.next_skip_count = 0 self.next_skip_timestamp = 0 # This validator's indices in the state self.indices = None # Is this validator active? self.active = False # Code that verifies signatures from this validator self.validation_code = generate_validation_code(privtoaddr(key)) # Parents that this validator has already built a block on self.used_parents = {} # This validator's clock offset (for testing purposes) self.time_offset = random.randrange(time_offset) - (time_offset // 2) # Determine the epoch length self.epoch_length = self.call_casper('getEpochLength') # Give this validator a unique ID self.id = len(ids) ids.append(self.id) self.find_my_indices() self.cached_head = self.chain.head_hash def call_casper(self, fun, args=[]): return call_casper(self.chain.state, fun, args) def find_my_indices(self): epoch = self.chain.state.block_number // self.epoch_length print 'Finding indices for epoch %d' % epoch, self.call_casper( 'getEpoch') for i in range(len(validator_sizes)): valcount = self.call_casper('getHistoricalValidatorCount', [epoch, i]) print i, valcount, self.call_casper('getHistoricalValidatorCount', [0, i]) for j in range(valcount): valcode = self.call_casper('getValidationCode', [i, j]) print(valcode, self.validation_code) if valcode == self.validation_code: self.indices = i, j start = self.call_casper('getStartEpoch', [i, j]) end = self.call_casper('getEndEpoch', [i, j]) if start <= epoch < end: self.active = True self.next_skip_count = 0 self.next_skip_timestamp = get_timestamp( self.chain, self.next_skip_count) print 'In current validator set at (%d, %d)' % (i, j) return else: self.indices = None self.active = False self.next_skip_count, self.next_skip_timestamp = 0, 0 print 'Registered at (%d, %d) but not in current set' % ( i, j) return self.indices = None self.active = False self.next_skip_count, self.next_skip_timestamp = 0, 0 print 'Not in current validator set' def get_uncles(self): anc = self.chain.get_block( self.chain.get_blockhash_by_number(self.chain.state.block_number - CHECK_FOR_UNCLES_BACK)) if anc: descendants = self.chain.get_descendants(anc) else: descendants = self.chain.get_descendants( self.chain.db.get('GENESIS_HASH')) potential_uncles = [ x for x in descendants if x not in self.chain and isinstance(x, Block) ] uncles = [ x.header for x in potential_uncles if not call_casper( self.chain.state, 'isDunkleIncluded', [x.header.hash]) ] return uncles def get_timestamp(self): return int(self.network.time * 0.01) + self.time_offset def on_receive(self, obj): if isinstance(obj, list): for _obj in obj: self.on_receive(_obj) return if obj.hash in self.received_objects: return if isinstance(obj, Block): print 'Receiving block', obj assert obj.hash not in self.chain block_success = self.chain.add_block(obj) self.network.broadcast(self, obj) self.network.broadcast(self, ChildRequest(obj.header.hash)) self.update_head() elif isinstance(obj, Transaction): if self.chain.add_transaction(obj): self.network.broadcast(self, obj) self.received_objects[obj.hash] = True for x in self.chain.get_chain(): assert x.hash in self.received_objects def tick(self): # Try to create a block # Conditions: # (i) you are an active validator, # (ii) you have not yet made a block with this parent if self.indices and self.chain.head_hash not in self.used_parents: t = self.get_timestamp() # Is it early enough to create the block? if t >= self.next_skip_timestamp and ( not self.chain.head or t > self.chain.head.header.timestamp): # Wrong validator; in this case, just wait for the next skip count if not check_skips(self.chain, self.indices, self.next_skip_count): self.next_skip_count += 1 self.next_skip_timestamp = get_timestamp( self.chain, self.next_skip_count) # print 'Incrementing proposed timestamp for block %d to %d' % \ # (self.chain.head.header.number + 1 if self.chain.head else 0, self.next_skip_timestamp) return self.used_parents[self.chain.head_hash] = True # Simulated 15% chance of validator failure to make a block if random.random() > 0.999: print 'Simulating validator failure, block %d not created' % ( self.chain.head.header.number + 1 if self.chain.head else 0) return # Make the block, make sure it's valid pre_dunkle_count = call_casper(self.chain.state, 'getTotalDunklesIncluded', []) dunkle_txs = [] for i, u in enumerate(self.get_uncles()[:4]): start_nonce = self.chain.state.get_nonce(self.address) txdata = casper_ct.encode('includeDunkle', [rlp.encode(u)]) dunkle_txs.append( Transaction(start_nonce + i, 0, 650000, self.chain.config['CASPER_ADDR'], 0, txdata).sign(self.key)) for dtx in dunkle_txs[::-1]: self.chain.add_transaction(dtx, force=True) blk = make_block(self.chain, self.key, self.randao, self.indices, self.next_skip_count) global global_block_counter global_block_counter += 1 for dtx in dunkle_txs: assert dtx in blk.transactions, (dtx, blk.transactions) print 'made block with timestamp %d and %d dunkles' % ( blk.timestamp, len(dunkle_txs)) assert blk.timestamp >= self.next_skip_timestamp assert self.chain.add_block(blk) self.update_head() post_dunkle_count = call_casper(self.chain.state, 'getTotalDunklesIncluded', []) assert post_dunkle_count - pre_dunkle_count == len(dunkle_txs) self.received_objects[blk.hash] = True print 'Validator %d making block %d (%s)' % ( self.id, blk.header.number, blk.header.hash[:8].encode('hex')) self.network.broadcast(self, blk) # Sometimes we received blocks too early or out of order; # run an occasional loop that processes these if random.random() < 0.02: self.chain.process_time_queue() self.chain.process_parent_queue() self.update_head() def update_head(self): if self.cached_head == self.chain.head_hash: return self.cached_head = self.chain.head_hash if self.chain.state.block_number % self.epoch_length == 0: self.find_my_indices() if self.indices: self.next_skip_count = 0 self.next_skip_timestamp = get_timestamp(self.chain, self.next_skip_count) print 'Head changed: %s, will attempt creating a block at %d' % ( self.chain.head_hash.encode('hex'), self.next_skip_timestamp) def withdraw(self, gasprice=20 * 10**9): h = sha3(b'withdrawwithdrawwithdrawwithdraw') v, r, s = ecsign(h, self.key) sigdata = encode_int32(v) + encode_int32(r) + encode_int32(s) txdata = casper_ct.encode('startWithdrawal', [self.indices[0], self.indices[1], sigdata]) tx = Transaction(self.chain.state.get_nonce(self.address), gasprice, 650000, self.chain.config['CASPER_ADDR'], 0, txdata).sign(self.key) self.chain.add_transaction(tx) self.network.broadcast(self, tx) print 'Withdrawing!'
class ChainService(WiredService): """ Manages the chain and requests to it. """ # required by BaseService name = 'chain' default_config = dict(eth=dict(network_id=0, genesis='', pruning=-1), block=ethereum_config.default_config) # required by WiredService wire_protocol = eth_protocol.ETHProtocol # create for each peer # initialized after configure: chain = None genesis = None synchronizer = None config = None block_queue_size = 1024 transaction_queue_size = 1024 processed_gas = 0 processed_elapsed = 0 def __init__(self, app): self.config = app.config sce = self.config['eth'] if int(sce['pruning']) >= 0: self.db = RefcountDB(app.services.db) if "I am not pruning" in self.db.db: raise Exception("This database was initialized as non-pruning." " Kinda hard to start pruning now.") self.db.ttl = int(sce['pruning']) self.db.db.put("I am pruning", "1") else: self.db = app.services.db if "I am pruning" in self.db: raise Exception("This database was initialized as pruning." " Kinda hard to stop pruning now.") self.db.put("I am not pruning", "1") if 'network_id' in self.db: db_network_id = self.db.get('network_id') if db_network_id != str(sce['network_id']): raise Exception( "This database was initialized with network_id {} " "and can not be used when connecting to network_id {}". format(db_network_id, sce['network_id'])) else: self.db.put('network_id', str(sce['network_id'])) self.db.commit() assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') coinbase = app.services.accounts.coinbase env = Env(self.db, sce['block']) self.chain = Chain(env, new_head_cb=self._on_new_head, coinbase=coinbase) log.info('chain at', number=self.chain.head.number) if 'genesis_hash' in sce: assert sce['genesis_hash'] == self.chain.genesis.hex_hash() self.synchronizer = Synchronizer(self, force_sync=None) self.block_queue = Queue(maxsize=self.block_queue_size) self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.add_transaction_lock = gevent.lock.Semaphore() self.broadcast_filter = DuplicatesFilter() self.on_new_head_cbs = [] self.on_new_head_candidate_cbs = [] self.newblock_processing_times = deque(maxlen=1000) @property def is_syncing(self): return self.synchronizer.synctask is not None @property def is_mining(self): if 'pow' in self.app.services: return self.app.services.pow.active return False def _on_new_head(self, block): # DEBUG('new head cbs', len(self.on_new_head_cbs)) for cb in self.on_new_head_cbs: cb(block) self._on_new_head_candidate( ) # we implicitly have a new head_candidate def _on_new_head_candidate(self): # DEBUG('new head candidate cbs', len(self.on_new_head_candidate_cbs)) for cb in self.on_new_head_candidate_cbs: cb(self.chain.head_candidate) def add_transaction(self, tx, origin=None): if self.is_syncing: return # we can not evaluate the tx based on outdated state log.debug('add_transaction', locked=self.add_transaction_lock.locked(), tx=tx) assert isinstance(tx, Transaction) assert origin is None or isinstance(origin, BaseProtocol) if tx.hash in self.broadcast_filter: log.debug('discarding known tx') # discard early return # validate transaction try: validate_transaction(self.chain.head_candidate, tx) log.debug('valid tx, broadcasting') self.broadcast_transaction(tx, origin=origin) # asap except InvalidTransaction as e: log.debug('invalid tx', error=e) return if origin is not None: # not locally added via jsonrpc if not self.is_mining or self.is_syncing: log.debug('discarding tx', syncing=self.is_syncing, mining=self.is_mining) return self.add_transaction_lock.acquire() success = self.chain.add_transaction(tx) self.add_transaction_lock.release() if success: self._on_new_head_candidate() def add_block(self, t_block, proto): "adds a block to the block_queue and spawns _add_block if not running" self.block_queue.put((t_block, proto)) # blocks if full if not self.add_blocks_lock: self.add_blocks_lock = True # need to lock here (ctx switch is later) gevent.spawn(self._add_blocks) def add_mined_block(self, block): log.debug('adding mined block', block=block) assert isinstance(block, Block) assert block.header.check_pow() if self.chain.add_block(block): log.debug('added', block=block, ts=time.time()) assert block == self.chain.head self.broadcast_newblock(block, chain_difficulty=block.chain_difficulty()) def knows_block(self, block_hash): "if block is in chain or in queue" if block_hash in self.chain: return True # check if queued or processed for i in range(len(self.block_queue.queue)): if block_hash == self.block_queue.queue[i][0].header.hash: return True return False def _add_blocks(self): log.debug('add_blocks', qsize=self.block_queue.qsize(), add_tx_lock=self.add_transaction_lock.locked()) assert self.add_blocks_lock is True self.add_transaction_lock.acquire() try: while not self.block_queue.empty(): t_block, proto = self.block_queue.peek( ) # peek: knows_block while processing if t_block.header.hash in self.chain: log.warn('known block', block=t_block) self.block_queue.get() continue if t_block.header.prevhash not in self.chain: log.warn('missing parent', block=t_block, head=self.chain.head) self.block_queue.get() continue # FIXME, this is also done in validation and in synchronizer for new_blocks if not t_block.header.check_pow(): log.warn('invalid pow', block=t_block, FIXME='ban node') sentry.warn_invalid(t_block, 'InvalidBlockNonce') self.block_queue.get() continue try: # deserialize st = time.time() block = t_block.to_block(env=self.chain.env) elapsed = time.time() - st log.debug('deserialized', elapsed='%.4fs' % elapsed, ts=time.time(), gas_used=block.gas_used, gpsec=self.gpsec(block.gas_used, elapsed)) except processblock.InvalidTransaction as e: log.warn('invalid transaction', block=t_block, error=e, FIXME='ban node') errtype = \ 'InvalidNonce' if isinstance(e, InvalidNonce) else \ 'NotEnoughCash' if isinstance(e, InsufficientBalance) else \ 'OutOfGasBase' if isinstance(e, InsufficientStartGas) else \ 'other_transaction_error' sentry.warn_invalid(t_block, errtype) self.block_queue.get() continue except VerificationFailed as e: log.warn('verification failed', error=e, FIXME='ban node') sentry.warn_invalid(t_block, 'other_block_error') self.block_queue.get() continue # Check canary score = 0 for address in canary_addresses: if block.get_storage_data(address, 1) > 0: score += 1 if score >= 2: log.warn('canary triggered') continue # All checks passed log.debug('adding', block=block, ts=time.time()) if self.chain.add_block( block, forward_pending_transactions=self.is_mining): now = time.time() log.info('added', block=block, txs=block.transaction_count, gas_used=block.gas_used) if t_block.newblock_timestamp: total = now - t_block.newblock_timestamp self.newblock_processing_times.append(total) avg = statistics.mean(self.newblock_processing_times) med = statistics.median(self.newblock_processing_times) max_ = max(self.newblock_processing_times) min_ = min(self.newblock_processing_times) log.info('processing time', last=total, avg=avg, max=max_, min=min_, median=med) else: log.warn('could not add', block=block) self.block_queue.get( ) # remove block from queue (we peeked only) gevent.sleep(0.001) finally: self.add_blocks_lock = False self.add_transaction_lock.release() def gpsec(self, gas_spent=0, elapsed=0): if gas_spent: self.processed_gas += gas_spent self.processed_elapsed += elapsed return int(self.processed_gas / (0.001 + self.processed_elapsed)) def broadcast_newblock(self, block, chain_difficulty=None, origin=None): if not chain_difficulty: assert block.hash in self.chain chain_difficulty = block.chain_difficulty() assert isinstance(block, (eth_protocol.TransientBlock, Block)) if self.broadcast_filter.update(block.header.hash): log.debug('broadcasting newblock', origin=origin) bcast = self.app.services.peermanager.broadcast bcast(eth_protocol.ETHProtocol, 'newblock', args=(block, chain_difficulty), exclude_peers=[origin.peer] if origin else []) else: log.debug('already broadcasted block') def broadcast_transaction(self, tx, origin=None): assert isinstance(tx, Transaction) if self.broadcast_filter.update(tx.hash): log.debug('broadcasting tx', origin=origin) bcast = self.app.services.peermanager.broadcast bcast(eth_protocol.ETHProtocol, 'transactions', args=(tx, ), exclude_peers=[origin.peer] if origin else []) else: log.debug('already broadcasted tx') # wire protocol receivers ########### def on_wire_protocol_start(self, proto): log.debug('----------------------------------') log.debug('on_wire_protocol_start', proto=proto) assert isinstance(proto, self.wire_protocol) # register callbacks proto.receive_status_callbacks.append(self.on_receive_status) proto.receive_transactions_callbacks.append( self.on_receive_transactions) proto.receive_getblockhashes_callbacks.append( self.on_receive_getblockhashes) proto.receive_blockhashes_callbacks.append(self.on_receive_blockhashes) proto.receive_getblocks_callbacks.append(self.on_receive_getblocks) proto.receive_blocks_callbacks.append(self.on_receive_blocks) proto.receive_newblock_callbacks.append(self.on_receive_newblock) proto.receive_newblockhashes_callbacks.append(self.on_newblockhashes) # send status head = self.chain.head proto.send_status(chain_difficulty=head.chain_difficulty(), chain_head_hash=head.hash, genesis_hash=self.chain.genesis.hash) def on_wire_protocol_stop(self, proto): assert isinstance(proto, self.wire_protocol) log.debug('----------------------------------') log.debug('on_wire_protocol_stop', proto=proto) def on_receive_status(self, proto, eth_version, network_id, chain_difficulty, chain_head_hash, genesis_hash): log.debug('----------------------------------') log.debug('status received', proto=proto, eth_version=eth_version) assert eth_version == proto.version, (eth_version, proto.version) if network_id != self.config['eth'].get('network_id', proto.network_id): log.warn("invalid network id", remote_network_id=network_id, expected_network_id=self.config['eth'].get( 'network_id', proto.network_id)) raise eth_protocol.ETHProtocolError('wrong network_id') # check genesis if genesis_hash != self.chain.genesis.hash: log.warn("invalid genesis hash", remote_id=proto, genesis=genesis_hash.encode('hex')) raise eth_protocol.ETHProtocolError('wrong genesis block') # request chain self.synchronizer.receive_status(proto, chain_head_hash, chain_difficulty) # send transactions transactions = self.chain.get_transactions() if transactions: log.debug("sending transactions", remote_id=proto) proto.send_transactions(*transactions) # transactions def on_receive_transactions(self, proto, transactions): "receives rlp.decoded serialized" log.debug('----------------------------------') log.debug('remote_transactions_received', count=len(transactions), remote_id=proto) for tx in transactions: self.add_transaction(tx, origin=proto) # blockhashes ########### def on_newblockhashes(self, proto, newblockhashes): """ msg sent out if not the full block is propagated chances are high, that we get the newblock, though. """ log.debug('----------------------------------') log.debug("recv newnewblockhashes", num=len(newblockhashes), remote_id=proto) assert len(newblockhashes) <= 32 self.synchronizer.receive_newblockhashes(proto, newblockhashes) def on_receive_getblockhashes(self, proto, child_block_hash, count): log.debug('----------------------------------') log.debug("handle_get_blockhashes", count=count, block_hash=encode_hex(child_block_hash)) max_hashes = min(count, self.wire_protocol.max_getblockhashes_count) found = [] if child_block_hash not in self.chain: log.debug("unknown block") proto.send_blockhashes(*[]) return last = child_block_hash while len(found) < max_hashes: try: last = rlp.decode_lazy( self.chain.db.get(last))[0][0] # [head][prevhash] except KeyError: # this can happen if we started a chain download, which did not complete # should not happen if the hash is part of the canonical chain log.warn('KeyError in getblockhashes', hash=last) break if last: found.append(last) else: break log.debug("sending: found block_hashes", count=len(found)) proto.send_blockhashes(*found) def on_receive_blockhashes(self, proto, blockhashes): log.debug('----------------------------------') if blockhashes: log.debug("on_receive_blockhashes", count=len(blockhashes), remote_id=proto, first=encode_hex(blockhashes[0]), last=encode_hex(blockhashes[-1])) else: log.debug("recv 0 remote block hashes, signifying genesis block") self.synchronizer.receive_blockhashes(proto, blockhashes) # blocks ################ def on_receive_getblocks(self, proto, blockhashes): log.debug('----------------------------------') log.debug("on_receive_getblocks", count=len(blockhashes)) found = [] for bh in blockhashes[:self.wire_protocol.max_getblocks_count]: try: found.append(self.chain.db.get(bh)) except KeyError: log.debug("unknown block requested", block_hash=encode_hex(bh)) if found: log.debug("found", count=len(found)) proto.send_blocks(*found) def on_receive_blocks(self, proto, transient_blocks): log.debug('----------------------------------') blk_number = max(x.header.number for x in transient_blocks) if transient_blocks else 0 log.debug("recv blocks", count=len(transient_blocks), remote_id=proto, highest_number=blk_number) if transient_blocks: self.synchronizer.receive_blocks(proto, transient_blocks) def on_receive_newblock(self, proto, block, chain_difficulty): log.debug('----------------------------------') log.debug("recv newblock", block=block, remote_id=proto) self.synchronizer.receive_newblock(proto, block, chain_difficulty) def on_receive_getblockheaders(self, proto, blockhashes): log.debug('----------------------------------') log.debug("on_receive_getblockheaders", count=len(blockhashes)) found = [] for bh in blockhashes[:self.wire_protocol.max_getblocks_count]: try: found.append(rlp.encode(rlp.decode(self.chain.db.get(bh))[0])) except KeyError: log.debug("unknown block requested", block_hash=encode_hex(bh)) if found: log.debug("found", count=len(found)) proto.send_blockheaders(*found) def on_receive_blockheaders(self, proto, transient_blocks): log.debug('----------------------------------') pass # TODO: implement headers first syncing def on_receive_hashlookup(self, proto, hashes): found = [] for h in hashes: try: found.append( utils.encode_hex( self.chain.db.get('node:' + utils.decode_hex(h)))) except KeyError: found.append('') proto.send_hashlookupresponse(h) def on_receive_hashlookupresponse(self, proto, hashresponses): pass
class ChainService(WiredService): """ Manages the chain and requests to it. """ # required by BaseService name = 'chain' default_config = dict(eth=dict(network_id=0)) # required by WiredService wire_protocol = eth_protocol.ETHProtocol # create for each peer # initialized after configure: chain = None genesis = None synchronizer = None config = None block_queue_size = 1024 transaction_queue_size = 1024 processed_gas = 0 processed_elapsed = 0 def __init__(self, app): self.config = app.config self.db = app.services.db assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') coinbase = app.services.accounts.coinbase self.chain = Chain(self.db, new_head_cb=self._on_new_head, coinbase=coinbase) log.info('chain at', number=self.chain.head.number) self.synchronizer = Synchronizer(self, force_sync=None) self.block_queue = Queue(maxsize=self.block_queue_size) self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.add_transaction_lock = gevent.lock.Semaphore() self.broadcast_filter = DuplicatesFilter() self.on_new_head_cbs = [] self.on_new_head_candidate_cbs = [] @property def is_syncing(self): return self.synchronizer.synctask is not None def _on_new_head(self, block): for cb in self.on_new_head_cbs: cb(block) self._on_new_head_candidate( ) # we implicitly have a new head_candidate def _on_new_head_candidate(self): for cb in self.on_new_head_candidate_cbs: cb(self.chain.head_candidate) def add_transaction(self, tx, origin=None): assert isinstance(tx, Transaction) log.debug('add_transaction', locked=self.add_transaction_lock.locked()) self.add_transaction_lock.acquire() success = self.chain.add_transaction(tx) self.add_transaction_lock.release() if success: self._on_new_head_candidate() self.broadcast_transaction(tx, origin=origin) # asap def add_block(self, t_block, proto): "adds a block to the block_queue and spawns _add_block if not running" self.block_queue.put((t_block, proto)) # blocks if full if not self.add_blocks_lock: self.add_blocks_lock = True # need to lock here (ctx switch is later) gevent.spawn(self._add_blocks) def add_mined_block(self, block): log.debug('adding mined block', block=block) assert block.check_pow() if self.chain.add_block(block): log.info('added', block=block, ts=time.time()) assert block == self.chain.head self.broadcast_newblock(block, chain_difficulty=block.chain_difficulty()) def knows_block(self, block_hash): "if block is in chain or in queue" if block_hash in self.chain: return True # check if queued or processed for i in range(len(self.block_queue.queue)): if block_hash == self.block_queue.queue[i][0].header.hash: return True return False def _add_blocks(self): log.debug('add_blocks', qsize=self.block_queue.qsize(), add_tx_lock=self.add_transaction_lock.locked()) assert self.add_blocks_lock is True self.add_transaction_lock.acquire() try: while not self.block_queue.empty(): t_block, proto = self.block_queue.peek( ) # peek: knows_block while processing if t_block.header.hash in self.chain: log.warn('known block', block=t_block) self.block_queue.get() continue if t_block.header.prevhash not in self.chain: log.warn('missing parent', block=t_block) self.block_queue.get() continue # FIXME, this is also done in validation and in synchronizer for new_blocks if not t_block.header.check_pow(): log.warn('invalid pow', block=t_block, FIXME='ban node') self.block_queue.get() continue try: # deserialize st = time.time() block = t_block.to_block(db=self.chain.db) elapsed = time.time() - st log.debug('deserialized', elapsed='%.4fs' % elapsed, gas_used=block.gas_used, gpsec=self.gpsec(block.gas_used, elapsed)) except processblock.InvalidTransaction as e: log.warn('invalid transaction', block=t_block, error=e, FIXME='ban node') self.block_queue.get() continue except VerificationFailed as e: log.warn('verification failed', error=e, FIXME='ban node') self.block_queue.get() continue if self.chain.add_block(block): log.info('added', block=block, ts=time.time()) self.block_queue.get( ) # remove block from queue (we peeked only) gevent.sleep(0.001) finally: self.add_blocks_lock = False self.add_transaction_lock.release() def gpsec(self, gas_spent=0, elapsed=0): self.processed_gas += gas_spent self.processed_elapsed += elapsed return int(self.processed_gas / (0.001 + self.processed_elapsed)) def broadcast_newblock(self, block, chain_difficulty=None, origin=None): if not chain_difficulty: assert block.hash in self.chain chain_difficulty = block.chain_difficulty() assert isinstance(block, (eth_protocol.TransientBlock, Block)) if self.broadcast_filter.known(block.header.hash): log.debug('already broadcasted block') else: log.debug('broadcasting newblock', origin=origin) bcast = self.app.services.peermanager.broadcast bcast(eth_protocol.ETHProtocol, 'newblock', args=(block, chain_difficulty), exclude_peers=[origin.peer] if origin else []) def broadcast_transaction(self, tx, origin=None): assert isinstance(tx, Transaction) if self.broadcast_filter.known(tx.hash): log.debug('already broadcasted tx') else: log.debug('broadcasting tx', origin=origin) bcast = self.app.services.peermanager.broadcast bcast(eth_protocol.ETHProtocol, 'transactions', args=(tx, ), exclude_peers=[origin.peer] if origin else []) # wire protocol receivers ########### def on_wire_protocol_start(self, proto): log.debug('on_wire_protocol_start', proto=proto) assert isinstance(proto, self.wire_protocol) # register callbacks proto.receive_status_callbacks.append(self.on_receive_status) proto.receive_transactions_callbacks.append( self.on_receive_transactions) proto.receive_getblockhashes_callbacks.append( self.on_receive_getblockhashes) proto.receive_blockhashes_callbacks.append(self.on_receive_blockhashes) proto.receive_getblocks_callbacks.append(self.on_receive_getblocks) proto.receive_blocks_callbacks.append(self.on_receive_blocks) proto.receive_newblock_callbacks.append(self.on_receive_newblock) # send status head = self.chain.head proto.send_status(chain_difficulty=head.chain_difficulty(), chain_head_hash=head.hash, genesis_hash=self.chain.genesis.hash) def on_wire_protocol_stop(self, proto): assert isinstance(proto, self.wire_protocol) log.debug('on_wire_protocol_stop', proto=proto) def on_receive_status(self, proto, eth_version, network_id, chain_difficulty, chain_head_hash, genesis_hash): log.debug('status received', proto=proto, eth_version=eth_version) assert eth_version == proto.version, (eth_version, proto.version) if network_id != self.config['eth'].get('network_id', proto.network_id): log.warn("invalid network id", remote_network_id=network_id, expected_network_id=self.config['eth'].get( 'network_id', proto.network_id)) raise eth_protocol.ETHProtocolError('wrong network_id') # check genesis if genesis_hash != self.chain.genesis.hash: log.warn("invalid genesis hash", remote_id=proto, genesis=genesis_hash.encode('hex')) raise eth_protocol.ETHProtocolError('wrong genesis block') # request chain self.synchronizer.receive_status(proto, chain_head_hash, chain_difficulty) # send transactions transactions = self.chain.get_transactions() if transactions: log.debug("sending transactions", remote_id=proto) proto.send_transactions(*transactions) # transactions def on_receive_transactions(self, proto, transactions): "receives rlp.decoded serialized" log.debug('remote_transactions_received', count=len(transactions), remote_id=proto) for tx in transactions: self.add_transaction(tx, origin=proto) # blockhashes ########### def on_receive_getblockhashes(self, proto, child_block_hash, count): log.debug("handle_get_blockhashes", count=count, block_hash=encode_hex(child_block_hash)) max_hashes = min(count, self.wire_protocol.max_getblockhashes_count) found = [] if child_block_hash not in self.chain: log.debug("unknown block") proto.send_blockhashes(*[]) return last = child_block_hash while len(found) < max_hashes: try: last = rlp.decode_lazy(self.chain.db.get(last))[0][0] except KeyError: # this can happen if we started a chain download, which did not complete # should not happen if the hash is part of the canonical chain log.warn('KeyError in getblockhashes', hash=last) break if last: found.append(last) else: break log.debug("sending: found block_hashes", count=len(found)) proto.send_blockhashes(*found) def on_receive_blockhashes(self, proto, blockhashes): if blockhashes: log.debug("on_receive_blockhashes", count=len(blockhashes), remote_id=proto, first=encode_hex(blockhashes[0]), last=encode_hex(blockhashes[-1])) else: log.debug("recv 0 remote block hashes, signifying genesis block") self.synchronizer.receive_blockhashes(proto, blockhashes) # blocks ################ def on_receive_getblocks(self, proto, blockhashes): log.debug("on_receive_getblocks", count=len(blockhashes)) found = [] for bh in blockhashes[:self.wire_protocol.max_getblocks_count]: try: found.append(self.chain.db.get(bh)) except KeyError: log.debug("unknown block requested", block_hash=encode_hex(bh)) if found: log.debug("found", count=len(found)) proto.send_blocks(*found) def on_receive_blocks(self, proto, transient_blocks): blk_number = max(x.header.number for x in transient_blocks) if transient_blocks else 0 log.debug("recv blocks", count=len(transient_blocks), remote_id=proto, highest_number=blk_number) if transient_blocks: self.synchronizer.receive_blocks(proto, transient_blocks) def on_receive_newblock(self, proto, block, chain_difficulty): log.debug("recv newblock", block=block, remote_id=proto) self.synchronizer.receive_newblock(proto, block, chain_difficulty)
class Validator(): def __init__(self, genesis, key, network, env, time_offset=5): # Create a chain object self.chain = Chain(genesis, env=env) # Use the validator's time as the chain's time self.chain.time = lambda: self.get_timestamp() # My private key self.key = key # My address self.address = privtoaddr(key) # My randao self.randao = RandaoManager(sha3(self.key)) # Pointer to the test p2p network self.network = network # Record of objects already received and processed self.received_objects = {} # The minimum eligible timestamp given a particular number of skips self.next_skip_count = 0 self.next_skip_timestamp = 0 # This validator's indices in the state self.indices = None # Code that verifies signatures from this validator self.validation_code = generate_validation_code(privtoaddr(key)) # Parents that this validator has already built a block on self.used_parents = {} # This validator's clock offset (for testing purposes) self.time_offset = random.randrange(time_offset) - (time_offset // 2) # Give this validator a unique ID self.id = len(ids) ids.append(self.id) self.find_my_indices() self.cached_head = self.chain.head_hash def find_my_indices(self): for i in range(len(validatorSizes)): epoch = self.chain.state.block_number // EPOCH_LENGTH valcount = call_casper(self.chain.state, 'getHistoricalValidatorCount', [epoch, i]) for j in range(valcount): valcode = call_casper(self.chain.state, 'getValidationCode', [i, j]) if valcode == self.validation_code: self.indices = i, j self.next_skip_count = 0 self.next_skip_timestamp = get_timestamp(self.chain, self.next_skip_count) print 'In current validator set at (%d, %d)' % (i, j) return self.indices = None self.next_skip_count, self.next_skip_timestamp = 0, 0 print 'Not in current validator set' def get_uncles(self): anc = self.chain.get_block(self.chain.get_blockhash_by_number(self.chain.state.block_number - CHECK_FOR_UNCLES_BACK)) if anc: descendants = self.chain.get_descendants(anc) else: descendants = self.chain.get_descendants(self.chain.db.get('GENESIS_HASH')) potential_uncles = [x for x in descendants if x not in self.chain and isinstance(x, Block)] uncles = [x.header for x in potential_uncles if not call_casper(self.chain.state, 'isDunkleIncluded', [x.header.hash])] return uncles def get_timestamp(self): return int(self.network.time * 0.01) + self.time_offset def on_receive(self, obj): if isinstance(obj, list): for _obj in obj: self.on_receive(_obj) return if obj.hash in self.received_objects: return if isinstance(obj, Block): print 'Receiving block', obj assert obj.hash not in self.chain, (self.received_objects, obj.hash, [x.hash for x in self.chain.get_chain()]) block_success = self.chain.add_block(obj) self.network.broadcast(self, obj) self.network.broadcast(self, ChildRequest(obj.header.hash)) self.update_head() elif isinstance(obj, Transaction): self.chain.add_transaction(obj) self.received_objects[obj.hash] = True for x in self.chain.get_chain(): assert x.hash in self.received_objects def tick(self): # Try to create a block # Conditions: # (i) you are an active validator, # (ii) you have not yet made a block with this parent if self.indices and self.chain.head_hash not in self.used_parents: t = self.get_timestamp() # Is it early enough to create the block? if t >= self.next_skip_timestamp and (not self.chain.head or t > self.chain.head.header.timestamp): print 'creating', t, self.next_skip_timestamp # Wrong validator; in this case, just wait for the next skip count if not check_skips(self.chain, self.indices, self.next_skip_count): self.next_skip_count += 1 self.next_skip_timestamp = get_timestamp(self.chain, self.next_skip_count) print 'Incrementing proposed timestamp for block %d to %d' % \ (self.chain.head.header.number + 1 if self.chain.head else 0, self.next_skip_timestamp) return self.used_parents[self.chain.head_hash] = True # Simulated 15% chance of validator failure to make a block if random.random() > 0.999: print 'Simulating validator failure, block %d not created' % (self.chain.head.header.number + 1 if self.chain.head else 0) return # Make the block, make sure it's valid pre_dunkle_count = call_casper(self.chain.state, 'getTotalDunklesIncluded', []) dunkle_txs = [] for i, u in enumerate(self.get_uncles()[:4]): start_nonce = self.chain.state.get_nonce(self.address) print 'start_nonce', start_nonce txdata = casper_ct.encode('includeDunkle', [rlp.encode(u)]) dunkle_txs.append(Transaction(start_nonce + i, 0, 650000, self.chain.config['CASPER_ADDR'], 0, txdata).sign(self.key)) for dtx in dunkle_txs[::-1]: self.chain.add_transaction(dtx, force=True) blk = make_block(self.chain, self.key, self.randao, self.indices, self.next_skip_count) global global_block_counter global_block_counter += 1 for dtx in dunkle_txs: assert dtx in blk.transactions, (dtx, blk.transactions) print 'made block with timestamp %d and %d dunkles' % (blk.timestamp, len(dunkle_txs)) assert blk.timestamp >= self.next_skip_timestamp assert self.chain.add_block(blk) self.update_head() post_dunkle_count = call_casper(self.chain.state, 'getTotalDunklesIncluded', []) assert post_dunkle_count - pre_dunkle_count == len(dunkle_txs) self.received_objects[blk.hash] = True print 'Validator %d making block %d (%s)' % (self.id, blk.header.number, blk.header.hash[:8].encode('hex')) self.network.broadcast(self, blk) # Sometimes we received blocks too early or out of order; # run an occasional loop that processes these if random.random() < 0.02: self.chain.process_time_queue() self.chain.process_parent_queue() self.update_head() def update_head(self): if self.cached_head == self.chain.head_hash: return self.cached_head = self.chain.head_hash if self.chain.state.block_number % EPOCH_LENGTH == 0: self.find_my_indices() if self.indices: self.next_skip_count = 0 self.next_skip_timestamp = get_timestamp(self.chain, self.next_skip_count) print 'Head changed: %s, will attempt creating a block at %d' % (self.chain.head_hash.encode('hex'), self.next_skip_timestamp)
class ChainService(WiredService): """ Manages the chain and requests to it. """ # required by BaseService name = 'chain' default_config = dict(eth=dict(privkey_hex='')) # required by WiredService wire_protocol = eth_protocol.ETHProtocol # create for each peer # initialized after configure: chain = None genesis = None synchronizer = None config = None block_queue_size = 1024 transaction_queue_size = 1024 def __init__(self, app): self.config = app.config self.db = app.services.db assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') self.chain = Chain(self.db, new_head_cb=self._on_new_head) self.synchronizer = Synchronizer(self, force_sync=None) self.chain.coinbase = privtoaddr( self.config['eth']['privkey_hex'].decode('hex')) self.block_queue = Queue(maxsize=self.block_queue_size) self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.broadcast_filter = DuplicatesFilter() def _on_new_head(self, block): pass def add_block(self, t_block, proto): "adds a block to the block_queue and spawns _add_block if not running" self.block_queue.put((t_block, proto)) # blocks if full if not self.add_blocks_lock: self.add_blocks_lock = True gevent.spawn(self._add_blocks) def _add_blocks(self): log.debug('add_blocks', qsize=self.block_queue.qsize()) try: while not self.block_queue.empty(): t_block, proto = self.block_queue.get() if t_block.header.hash in self.chain: log.warn('known block', block=t_block) continue if t_block.header.prevhash not in self.chain: log.warn('missing parent', block=t_block) continue if not t_block.header.check_pow(): log.warn('invalid pow', block=t_block) # FIXME ban node continue try: # deserialize st = time.time() block = t_block.to_block(db=self.chain.db) elapsed = time.time() - st log.debug('deserialized', elapsed='%.2fs' % elapsed, gas_used=block.gas_used, gpsec=int(block.gas_used / elapsed)) except processblock.InvalidTransaction as e: log.warn('invalid transaction', block=t_block, error=e) # FIXME ban node continue if self.chain.add_block(block): log.debug('added', block=block) gevent.sleep(0.001) finally: self.add_blocks_lock = False def broadcast_newblock(self, block, chain_difficulty, origin=None): assert isinstance(block, eth_protocol.TransientBlock) if self.broadcast_filter.known(block.header.hash): log.debug('already broadcasted block') else: log.debug('broadcasting newblock', origin=origin) bcast = self.app.services.peermanager.broadcast bcast(eth_protocol.ETHProtocol, 'newblock', args=(block, chain_difficulty), num_peers=None, exclude_protos=[origin]) # wire protocol receivers ########### def on_wire_protocol_start(self, proto): log.debug('on_wire_protocol_start', proto=proto) assert isinstance(proto, self.wire_protocol) # register callbacks proto.receive_status_callbacks.append(self.on_receive_status) proto.receive_transactions_callbacks.append( self.on_receive_transactions) proto.receive_getblockhashes_callbacks.append( self.on_receive_getblockhashes) proto.receive_blockhashes_callbacks.append(self.on_receive_blockhashes) proto.receive_getblocks_callbacks.append(self.on_receive_getblocks) proto.receive_blocks_callbacks.append(self.on_receive_blocks) proto.receive_newblock_callbacks.append(self.on_receive_newblock) # send status head = self.chain.head proto.send_status(chain_difficulty=head.chain_difficulty(), chain_head_hash=head.hash, genesis_hash=self.chain.genesis.hash) def on_wire_protocol_stop(self, proto): assert isinstance(proto, self.wire_protocol) log.debug('on_wire_protocol_stop', proto=proto) def on_receive_status(self, proto, eth_version, network_id, chain_difficulty, chain_head_hash, genesis_hash): log.debug('status received', proto=proto, eth_version=eth_version) assert eth_version == proto.version, (eth_version, proto.version) if network_id != proto.network_id: log.warn("invalid network id", remote_id=proto.network_id, network_id=network_id) raise eth_protocol.ETHProtocolError('wrong network_id') # check genesis if genesis_hash != self.chain.genesis.hash: log.warn("invalid genesis hash", remote_id=proto, genesis=genesis_hash.encode('hex')) raise eth_protocol.ETHProtocolError('wrong genesis block') # request chain self.synchronizer.receive_status(proto, chain_head_hash, chain_difficulty) # send transactions transactions = self.chain.get_transactions() if transactions: log.debug("sending transactions", remote_id=proto) proto.send_transactions(*transactions) # transactions def on_receive_transactions(self, proto, transactions): "receives rlp.decoded serialized" log.debug('remote_transactions_received', count=len(transactions), remote_id=proto) log.debug('skipping, FIXME') return for tx in transactions: # fixme bloomfilter self.chain.add_transaction(tx) # blockhashes ########### def on_receive_getblockhashes(self, proto, child_block_hash, count): log.debug("handle_get_blockhashes", count=count, block_hash=encode_hex(child_block_hash)) max_hashes = min(count, self.wire_protocol.max_getblockhashes_count) found = [] if child_block_hash not in self.chain: log.debug("unknown block") proto.send_blockhashes(*[]) return last = child_block_hash while len(found) < max_hashes: last = rlp.decode_lazy(self.chain.db.get(last))[0][0] if last: found.append(last) else: break log.debug("sending: found block_hashes", count=len(found)) proto.send_blockhashes(*found) def on_receive_blockhashes(self, proto, blockhashes): if blockhashes: log.debug("on_receive_blockhashes", count=len(blockhashes), remote_id=proto, first=encode_hex(blockhashes[0]), last=encode_hex(blockhashes[-1])) else: log.debug("recv 0 remote block hashes, signifying genesis block") self.synchronizer.receive_blockhashes(proto, blockhashes) # blocks ################ def on_receive_getblocks(self, proto, blockhashes): log.debug("on_receive_getblocks", count=len(blockhashes)) found = [] for bh in blockhashes[:self.wire_protocol.max_getblocks_count]: try: found.append(self.chain.db.get(bh)) except KeyError: log.debug("unknown block requested", block_hash=encode_hex(bh)) if found: log.debug("found", count=len(found)) proto.send_blocks(*found) def on_receive_blocks(self, proto, transient_blocks): log.debug("recv blocks", count=len(transient_blocks), remote_id=proto, highest_number=max(x.header.number for x in transient_blocks)) if transient_blocks: self.synchronizer.receive_blocks(proto, transient_blocks) def on_receive_newblock(self, proto, block, chain_difficulty): log.debug("recv newblock", block=block, remote_id=proto) self.synchronizer.receive_newblock(proto, block, chain_difficulty)
class ChainService(WiredService): """ Manages the chain and requests to it. """ # required by BaseService name = 'chain' default_config = dict(eth=dict(privkey_hex='')) # required by WiredService wire_protocol = eth_protocol.ETHProtocol # create for each peer # initialized after configure: chain = None genesis = None miner = None synchronizer = None config = None def __init__(self, app): self.config = app.config self.db = app.services.db assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') self.chain = Chain(self.db, new_head_cb=self._on_new_head) self.synchronizer = Synchronizer(self.chain) self.chain.coinbase = privtoaddr(self.config['eth']['privkey_hex'].decode('hex')) def _on_new_head(self, block): self.miner = Miner(self.chain.head_candidate) # if we are not syncing, forward all blocks if not self.synchronizer.synchronization_tasks: log.debug("_on_new_head", block=block) # signals.broadcast_new_block.send(sender=None, block=block) # FIXME def loop_body(self): ts = time.time() pct_cpu = self.config['misc']['mining'] if pct_cpu > 0: self.mine() delay = (time.time() - ts) * (100. / pct_cpu - 1) assert delay >= 0 time.sleep(min(delay, 1.)) else: time.sleep(.01) def mine(self): block = self.miner.mine() if block: # create new block assert self.chain.add_block(block), ("newly mined block is invalid!?", block) def receive_chain(self, transient_blocks, proto=None, new_block=False): _db = EphemDB() # assuming to receive chain order w/ oldest block first transient_blocks.sort(key=lambda x: x.header.number) assert transient_blocks[0].header.number <= transient_blocks[-1].header.number # notify syncer self.synchronizer.received_blocks(proto, transient_blocks) for t_block in transient_blocks: # oldest to newest if t_block.header.hash in self.chain: log.debug('known', block=t_block) continue if t_block.header.prevhash not in self.chain: log.debug('unknown parent', block=t_block, parent=t_block.header.prevhash.encode('hex')) # FIXME: not properly handled if we receive a differnt chain? # no problem with new getBlocks? self.synchronizer.synchronize_unknown_block(proto, t_block.header.hash, t_block=t_block) continue log.debug('checking pow', block=t_block) if not t_block.header.check_pow(_db): log.warn('invalid pow', block=t_block, proto=proto) continue # BROADCAST HERE !!! log.debug('deserializing', block=t_block, gas_used=t_block.header.gas_used) if t_block.header.prevhash == self.chain.head.hash: log.trace('is child') if t_block.header.prevhash == self.chain.genesis.hash: log.trace('is child of genesis') try: # block = blocks.Block(t_block.header, t_block.transaction_list, t_block.uncles, # db=self.chain.db) st = time.time() block = t_block.to_block(db=self.chain.db) elapsed = time.time() - st log.debug('deserialized', elapsed='%.2fs' % elapsed, gas_used=block.gas_used, gpsec=int(block.gas_used / elapsed)) except processblock.InvalidTransaction as e: # FIXME there might be another exception in # blocks.deserializeChild when replaying transactions # if this fails, we need to rewind state log.warn('invalid transaction', block=t_block, error=e, proto=proto) # stop current syncing of this chain and skip the child blocks self.synchronizer.stop_synchronization(proto) return except blocks.UnknownParentException: # gets never called # FIXME log.debug('unknown parent', block=t_block) if t_block.header.prevhash == blocks.GENESIS_PREVHASH: log.warn('wrong genesis', block=t_block, proto=proto) if proto is not None: proto.send_disconnect(reason='Wrong genesis block') raise eth_protocol.ETHProtocolError('wrong genesis') else: # should be a single newly mined block assert t_block.header.prevhash not in self.chain if t_block.header.prevhash == self.chain.genesis.hash: print t_block.serialize().encode('hex') assert t_block.header.prevhash != self.chain.genesis.hash log.debug('unknown parent', block=t_block, parent_hash=encode_hex(t_block.header.prevhash), remote_id=proto) if len(transient_blocks) != 1: # strange situation here. # we receive more than 1 block, so it's not a single newly mined one # sync/network/... failed to add the needed parent at some point # well, this happens whenever we can't validate a block! # we should disconnect! log.warn( 'blocks received, but unknown parent.', num=len(transient_blocks)) if proto is not None: # request chain for newest known hash self.synchronizer.synchronize_unknown_block( proto, transient_blocks[-1].header.hash, t_block=transient_blocks[-1]) break if block.hash in self.chain: log.debug('known', block=block) else: assert block.has_parent() # assume single block is newly mined block old_head_num = self.chain.head.number success = self.chain.add_block(block) if success: log.debug('added', block=block) else: raise eth_protocol.ETHProtocolError('could not add block') # broadcast if block.number > old_head_num and new_block: log.debug('broadcasting new head', block=block) f = self.app.services.peermanager.broadcast f(eth_protocol.ETHProtocol, 'newblock', args=(block, block.chain_difficulty()), num_peers=None, exclude_protos=[proto]) # wire protocol receivers ########### def on_wire_protocol_start(self, proto): log.debug('on_wire_protocol_start', proto=proto) assert isinstance(proto, self.wire_protocol) # register callbacks proto.receive_status_callbacks.append(self.on_receive_status) proto.receive_transactions_callbacks.append(self.on_receive_transactions) proto.receive_getblockhashes_callbacks.append(self.on_receive_getblockhashes) proto.receive_blockhashes_callbacks.append(self.on_receive_blockhashes) proto.receive_getblocks_callbacks.append(self.on_receive_getblocks) proto.receive_blocks_callbacks.append(self.on_receive_blocks) proto.receive_newblock_callbacks.append(self.on_receive_newblock) # send status head = self.chain.head proto.send_status(total_difficulty=head.chain_difficulty(), chain_head_hash=head.hash, genesis_hash=self.chain.genesis.hash) def on_wire_protocol_stop(self, proto): assert isinstance(proto, self.wire_protocol) log.debug('on_wire_protocol_stop', proto=proto) def on_receive_status(self, proto, eth_version, network_id, total_difficulty, chain_head_hash, genesis_hash): log.debug('status received', proto=proto, eth_version=eth_version) assert eth_version == proto.version, (eth_version, proto.version) if network_id != proto.network_id: log.warn("invalid network id", remote_id=proto, network_id=network_id) raise eth_protocol.ETHProtocolError('wrong network_id') # check genesis if genesis_hash != self.chain.genesis.hash: log.warn("invalid genesis hash", remote_id=proto, genesis=genesis_hash.encode('hex')) raise eth_protocol.ETHProtocolError('wrong genesis block') # request chain self.synchronizer.synchronize_status(proto, chain_head_hash, total_difficulty) # send transactions log.debug("sending transactions", remote_id=proto) transactions = self.chain.get_transactions() proto.send_transactions(*transactions) # transactions def on_receive_transactions(self, proto, transactions): "receives rlp.decoded serialized" log.debug('remote_transactions_received', count=len(transactions), remote_id=proto) log.debug('skipping, FIXME') return for tx in transactions: # fixme bloomfilter self.chain.add_transaction(tx) # blockhashes ########### def on_receive_getblockhashes(self, proto, child_block_hash, count): log.debug("handle_get_block_hashes", count=count, block_hash=encode_hex(child_block_hash)) max_hashes = min(count, MAX_GET_CHAIN_SEND_HASHES) found = [] if child_block_hash not in self.chain: log.debug("unknown block") proto.send_blockhashes([]) last = self.chain.get(child_block_hash) while len(found) < max_hashes: if last.has_parent(): last = last.get_parent() found.append(last.hash) else: break log.debug("sending: found block_hashes", count=len(found)) proto.send_blockhashes(*found) def on_receive_blockhashes(self, proto, block_hashes): if block_hashes: log.debug("on_receive_blockhashes", count=len(block_hashes), remote_id=proto, first=encode_hex(block_hashes[0]), last=encode_hex(block_hashes[-1])) else: log.debug("recv 0 remote block hashes, signifying genesis block") self.synchronizer.received_block_hashes(proto, block_hashes) # blocks ################ def on_receive_getblocks(self, proto, block_hashes): log.debug("on_receive_getblocks", count=len(block_hashes)) found = [] for bh in block_hashes[:MAX_GET_CHAIN_REQUEST_BLOCKS]: if bh in self.chain: found.append(self.chain.get(bh)) else: log.debug("unknown block requested", block_hash=encode_hex(bh)) if found: log.debug("found", count=len(found), first=found[0].hex_hash()) proto.send_blocks(*found) def on_receive_blocks(self, proto, transient_blocks): log.debug("recv remote blocks", count=len(transient_blocks), remote_id=proto, highest_number=max(x.header.number for x in transient_blocks)) if transient_blocks: self.receive_chain(transient_blocks, proto) def on_receive_newblock(self, proto, block, total_difficulty): log.debug("recv new remote block", block=block, remote_id=proto) self.receive_chain([block], proto, new_block=True)