def test_mine_block_with_transaction2(db):
k, v, k2, v2 = accounts()
chain = Chain({v: {"balance": utils.denoms.ether * 1}}, difficulty=1)
genesis_hash = chain.state.prev_headers[0].hash
tx = get_transaction()
blk2 = mine_next_block(chain, coinbase=v, transactions=[tx])
assert tx in blk2.transactions
assert tx in blk2.transactions
assert chain.get_block(blk2.hash) == blk2
assert tx.gasprice == 0
assert chain.state.get_balance(
v) == chain.env.config['BLOCK_REWARD'] + chain.mk_poststate_of_blockhash(genesis_hash).get_balance(v) - tx.value
def test_genesis_chain(db):
k, v, k2, v2 = accounts()
chain = Chain({v: {"balance": utils.denoms.ether * 1}}, difficulty=1)
blk = mine_on_chain(chain)
print('blook', blk)
assert chain.has_block(blk.hash)
assert blk.hash in chain
assert chain.get_block(blk.hash) == blk
assert chain.head == blk
assert chain.get_children(blk) == []
assert chain.get_chain() == [blk]
assert chain.get_block_by_number(1)
assert not chain.get_block_by_number(2)
assert chain.get_block_by_number(1) == blk
def test_simple_chain(db):
k, v, k2, v2 = accounts()
chain = Chain({v: {"balance": utils.denoms.ether * 1}}, difficulty=1)
tx = get_transaction()
blk2 = mine_next_block(chain, transactions=[tx])
blk3 = mine_next_block(chain)
assert blk2.hash in chain
assert blk3.hash in chain
assert chain.has_block(blk2.hash)
assert chain.has_block(blk3.hash)
assert chain.get_block(blk2.hash) == blk2
assert chain.get_block(blk3.hash) == blk3
assert chain.head == blk3
assert chain.get_children(blk2) == [blk3]
assert chain.get_chain() == [blk2, blk3]
assert chain.get_block_by_number(1) == blk2
assert chain.get_block_by_number(2) == blk3
assert not chain.get_block_by_number(3)
assert chain.get_tx_position(tx.hash) == (blk2.number, 0)
class ChainService(WiredService):
"""
Manages the chain and requests to it.
"""
# required by BaseService
name = 'chain'
default_config = dict(eth=dict(network_id=0, genesis='', pruning=-1),
block=ethereum_config.default_config)
# required by WiredService
wire_protocol = eth_protocol.ETHProtocol # create for each peer
# initialized after configure:
chain = None
genesis = None
synchronizer = None
config = None
block_queue_size = 1024
processed_gas = 0
processed_elapsed = 0
process_time_queue_period = 5
def __init__(self, app):
self.config = app.config
sce = self.config['eth']
if int(sce['pruning']) >= 0:
self.db = RefcountDB(app.services.db)
if "I am not pruning" in self.db.db:
raise RuntimeError(
"The database in '{}' was initialized as non-pruning. "
"Can not enable pruning now.".format(
self.config['data_dir']))
self.db.ttl = int(sce['pruning'])
self.db.db.put("I am pruning", "1")
else:
self.db = app.services.db
if "I am pruning" in self.db:
raise RuntimeError(
"The database in '{}' was initialized as pruning. "
"Can not disable pruning now".format(
self.config['data_dir']))
self.db.put("I am not pruning", "1")
if 'network_id' in self.db:
db_network_id = self.db.get(b'network_id')
if db_network_id != to_string(sce['network_id']):
raise RuntimeError(
"The database in '{}' was initialized with network id {} and can not be used "
"when connecting to network id {}. Please choose a different data directory."
.format(self.config['data_dir'], db_network_id,
sce['network_id']))
else:
self.db.put(b'network_id', to_string(sce['network_id']))
self.db.commit()
assert self.db is not None
super(ChainService, self).__init__(app)
log.info('initializing chain')
coinbase = app.services.accounts.coinbase
env = Env(self.db, sce['block'])
genesis_data = sce.get('genesis_data', {})
if not genesis_data:
genesis_data = mk_genesis_data(env)
self.chain = Chain(env=env,
genesis=genesis_data,
coinbase=coinbase,
new_head_cb=self._on_new_head)
header = self.chain.state.prev_headers[0]
log.info('chain at', number=header.number)
if 'genesis_hash' in sce:
assert sce['genesis_hash'] == self.chain.genesis.hex_hash, \
"Genesis hash mismatch.\n Expected: %s\n Got: %s" % (
sce['genesis_hash'], self.chain.genesis.hex_hash)
self.dao_challenges = dict()
self.synchronizer = Synchronizer(self, force_sync=None)
self.block_queue = Queue(maxsize=self.block_queue_size)
# When the transaction_queue is modified, we must set
# self._head_candidate_needs_updating to True in order to force the
# head candidate to be updated.
self.transaction_queue = TransactionQueue()
self._head_candidate_needs_updating = True
# Initialize a new head candidate.
_ = self.head_candidate
self.min_gasprice = 20 * 10**9 # TODO: better be an option to validator service?
self.add_blocks_lock = False
self.add_transaction_lock = gevent.lock.Semaphore()
self.broadcast_filter = DuplicatesFilter()
self.on_new_head_cbs = []
self.newblock_processing_times = deque(maxlen=1000)
gevent.spawn_later(self.process_time_queue_period,
self.process_time_queue)
@property
def is_syncing(self):
return self.synchronizer.synctask is not None
@property
def is_mining(self):
if 'pow' in self.app.services:
return self.app.services.pow.active
if 'validator' in self.app.services:
return self.app.services.validator.active
return False
def process_time_queue(self):
try:
self.chain.process_time_queue()
except Exception as e:
log.info(str(e))
finally:
gevent.spawn_later(self.process_time_queue_period,
self.process_time_queue)
# TODO: Move to pyethereum
def get_receipts(self, block):
# Receipts are no longer stored in the database, so need to generate
# them on the fly here.
temp_state = self.chain.mk_poststate_of_blockhash(
block.header.prevhash)
initialize(temp_state, block)
for tx in block.transactions:
apply_transaction(temp_state, tx)
return temp_state.receipts
def _on_new_head(self, block):
log.debug('new head cbs', num=len(self.on_new_head_cbs))
self.transaction_queue = self.transaction_queue.diff(
block.transactions)
self._head_candidate_needs_updating = True
for cb in self.on_new_head_cbs:
cb(block)
@property
def head_candidate(self):
if self._head_candidate_needs_updating:
self._head_candidate_needs_updating = False
# Make a copy of self.transaction_queue because
# make_head_candidate modifies it.
txqueue = copy.deepcopy(self.transaction_queue)
self._head_candidate, self._head_candidate_state = make_head_candidate(
self.chain, txqueue, timestamp=int(time.time()))
return self._head_candidate
def add_transaction(self,
tx,
origin=None,
force_broadcast=False,
force=False):
if self.is_syncing:
if force_broadcast:
assert origin is None # only allowed for local txs
log.debug('force broadcasting unvalidated tx')
self.broadcast_transaction(tx, origin=origin)
return # we can not evaluate the tx based on outdated state
log.debug('add_transaction',
locked=(not self.add_transaction_lock.locked()),
tx=tx)
assert isinstance(tx, Transaction)
assert origin is None or isinstance(origin, BaseProtocol)
if tx.hash in self.broadcast_filter:
log.debug('discarding known tx') # discard early
return
# validate transaction
try:
# Transaction validation for broadcasting. Transaction is validated
# against the current head candidate.
validate_transaction(self._head_candidate_state, tx)
log.debug('valid tx, broadcasting')
self.broadcast_transaction(tx, origin=origin) # asap
except InvalidTransaction as e:
log.debug('invalid tx', error=e)
return
if origin is not None: # not locally added via jsonrpc
if not self.is_mining or self.is_syncing:
log.debug('discarding tx',
syncing=self.is_syncing,
mining=self.is_mining)
return
if tx.gasprice >= self.min_gasprice:
self.add_transaction_lock.acquire()
self.transaction_queue.add_transaction(tx, force=force)
self._head_candidate_needs_updating = True
self.add_transaction_lock.release()
else:
log.info("too low gasprice, ignore",
tx=encode_hex(tx.hash)[:8],
gasprice=tx.gasprice)
def check_header(self, header):
return check_pow(self.chain.state, header)
def add_block(self, t_block, proto):
"adds a block to the block_queue and spawns _add_block if not running"
self.block_queue.put((t_block, proto)) # blocks if full
if not self.add_blocks_lock:
self.add_blocks_lock = True # need to lock here (ctx switch is later)
gevent.spawn(self._add_blocks)
def add_mined_block(self, block):
log.debug('adding mined block', block=block)
assert isinstance(block, Block)
if self.chain.add_block(block):
log.debug('added', block=block, ts=time.time())
assert block == self.chain.head
self.transaction_queue = self.transaction_queue.diff(
block.transactions)
self._head_candidate_needs_updating = True
self.broadcast_newblock(
block, chain_difficulty=self.chain.get_score(block))
return True
log.debug('failed to add', block=block, ts=time.time())
return False
def knows_block(self, block_hash):
"if block is in chain or in queue"
if self.chain.has_blockhash(block_hash):
return True
# check if queued or processed
for i in range(len(self.block_queue.queue)):
if block_hash == self.block_queue.queue[i][0].header.hash:
return True
return False
def _add_blocks(self):
log.debug('add_blocks',
qsize=self.block_queue.qsize(),
add_tx_lock=self.add_transaction_lock.locked())
assert self.add_blocks_lock is True
self.add_transaction_lock.acquire()
try:
while not self.block_queue.empty():
# sleep at the beginning because continue keywords will skip bottom
gevent.sleep(0.001)
t_block, proto = self.block_queue.peek(
) # peek: knows_block while processing
if self.chain.has_blockhash(t_block.header.hash):
log.warn('known block', block=t_block)
self.block_queue.get()
continue
if not self.chain.has_blockhash(t_block.header.prevhash):
log.warn('missing parent',
block=t_block,
head=self.chain.head)
self.block_queue.get()
continue
try: # deserialize
st = time.time()
block = t_block.to_block()
elapsed = time.time() - st
log.debug('deserialized',
elapsed='%.4fs' % elapsed,
ts=time.time(),
gas_used=block.gas_used,
gpsec=self.gpsec(block.gas_used, elapsed))
except InvalidTransaction as e:
log.warn('invalid transaction',
block=t_block,
error=e,
FIXME='ban node')
errtype = \
'InvalidNonce' if isinstance(e, InvalidNonce) else \
'NotEnoughCash' if isinstance(e, InsufficientBalance) else \
'OutOfGasBase' if isinstance(e, InsufficientStartGas) else \
'other_transaction_error'
sentry.warn_invalid(t_block, errtype)
self.block_queue.get()
continue
except VerificationFailed as e:
log.warn('verification failed', error=e, FIXME='ban node')
sentry.warn_invalid(t_block, 'other_block_error')
self.block_queue.get()
continue
# All checks passed
log.debug('adding', block=block, ts=time.time())
if self.chain.add_block(block):
now = time.time()
log.info('added',
block=block,
txs=block.transaction_count,
gas_used=block.gas_used)
if t_block.newblock_timestamp:
total = now - t_block.newblock_timestamp
self.newblock_processing_times.append(total)
avg = statistics.mean(self.newblock_processing_times)
med = statistics.median(self.newblock_processing_times)
max_ = max(self.newblock_processing_times)
min_ = min(self.newblock_processing_times)
log.info('processing time',
last=total,
avg=avg,
max=max_,
min=min_,
median=med)
if self.is_mining:
self.transaction_queue = self.transaction_queue.diff(
block.transactions)
else:
log.warn('could not add', block=block)
self.block_queue.get(
) # remove block from queue (we peeked only)
finally:
self.add_blocks_lock = False
self.add_transaction_lock.release()
def gpsec(self, gas_spent=0, elapsed=0):
if gas_spent:
self.processed_gas += gas_spent
self.processed_elapsed += elapsed
return int(
old_div(self.processed_gas, (0.001 + self.processed_elapsed)))
def broadcast_newblock(self, block, chain_difficulty=None, origin=None):
if not chain_difficulty:
assert self.chain.has_blockhash(block.hash)
chain_difficulty = self.chain.get_score(block)
assert isinstance(block, (eth_protocol.TransientBlock, Block))
if self.broadcast_filter.update(block.header.hash):
log.debug('broadcasting newblock', origin=origin)
bcast = self.app.services.peermanager.broadcast
bcast(eth_protocol.ETHProtocol,
'newblock',
args=(block, chain_difficulty),
exclude_peers=[origin.peer] if origin else [])
else:
log.debug('already broadcasted block')
def broadcast_transaction(self, tx, origin=None):
assert isinstance(tx, Transaction)
if self.broadcast_filter.update(tx.hash):
log.debug('broadcasting tx', origin=origin)
bcast = self.app.services.peermanager.broadcast
bcast(eth_protocol.ETHProtocol,
'transactions',
args=(tx, ),
exclude_peers=[origin.peer] if origin else [])
else:
log.debug('already broadcasted tx')
def query_headers(self,
hash_mode,
max_hashes,
skip,
reverse,
origin_hash=None,
number=None):
headers = []
unknown = False
while not unknown and len(headers) < max_hashes:
if hash_mode:
if not origin_hash:
break
block = self.chain.get_block(origin_hash)
if not block:
break
# If reached genesis, stop
if block.number == 0:
break
origin = block.header
else:
# If reached genesis, stop
if number is None or number == 0:
break
block = self.chain.get_block_by_number(number)
if block is None:
break
origin = block.header
headers.append(origin)
if hash_mode: # hash traversal
if reverse:
for i in range(skip + 1):
try:
block = self.chain.get_block(origin_hash)
if block:
origin_hash = block.prevhash
else:
unknown = True
break
except KeyError:
unknown = True
break
else:
blockhash = self.chain.get_blockhash_by_number(
origin.number + skip + 1)
try:
# block = self.chain.get_block(blockhash)
if block and self.chain.get_blockhashes_from_hash(
blockhash, skip + 1)[skip] == origin_hash:
origin_hash = blockhash
else:
unknown = True
except KeyError:
unknown = True
else: # number traversal
if reverse:
if number >= (skip + 1):
number -= (skip + 1)
else:
unknown = True
else:
number += (skip + 1)
return headers
# wire protocol receivers ###########
def on_wire_protocol_start(self, proto):
log.debug('----------------------------------')
log.debug('on_wire_protocol_start', proto=proto)
assert isinstance(proto, self.wire_protocol)
# register callbacks
proto.receive_status_callbacks.append(self.on_receive_status)
proto.receive_newblockhashes_callbacks.append(self.on_newblockhashes)
proto.receive_transactions_callbacks.append(
self.on_receive_transactions)
proto.receive_getblockheaders_callbacks.append(
self.on_receive_getblockheaders)
proto.receive_blockheaders_callbacks.append(
self.on_receive_blockheaders)
proto.receive_getblockbodies_callbacks.append(
self.on_receive_getblockbodies)
proto.receive_blockbodies_callbacks.append(self.on_receive_blockbodies)
proto.receive_newblock_callbacks.append(self.on_receive_newblock)
# send status
head = self.chain.head
proto.send_status(chain_difficulty=self.chain.get_score(head),
chain_head_hash=head.hash,
genesis_hash=self.chain.genesis.hash)
def on_wire_protocol_stop(self, proto):
assert isinstance(proto, self.wire_protocol)
log.debug('----------------------------------')
log.debug('on_wire_protocol_stop', proto=proto)
def on_receive_status(self, proto, eth_version, network_id,
chain_difficulty, chain_head_hash, genesis_hash):
log.debug('----------------------------------')
log.debug('status received', proto=proto, eth_version=eth_version)
if eth_version != proto.version:
if ('eth', proto.version) in proto.peer.remote_capabilities:
# if remote peer is capable of our version, keep the connection
# even the peer tried a different version
pass
else:
log.debug("no capable protocol to use, disconnect",
proto=proto,
eth_version=eth_version)
proto.send_disconnect(proto.disconnect.reason.useless_peer)
return
if network_id != self.config['eth'].get('network_id',
proto.network_id):
log.debug("invalid network id",
remote_network_id=network_id,
expected_network_id=self.config['eth'].get(
'network_id', proto.network_id))
raise eth_protocol.ETHProtocolError('wrong network_id')
# check genesis
if genesis_hash != self.chain.genesis.hash:
log.warn("invalid genesis hash",
remote_id=proto,
genesis=encode_hex(genesis_hash))
raise eth_protocol.ETHProtocolError('wrong genesis block')
# initiate DAO challenge
self.dao_challenges[proto] = (DAOChallenger(self, proto),
chain_head_hash, chain_difficulty)
def on_dao_challenge_answer(self, proto, result):
if result:
log.debug("DAO challenge passed")
_, chain_head_hash, chain_difficulty = self.dao_challenges[proto]
# request chain
self.synchronizer.receive_status(proto, chain_head_hash,
chain_difficulty)
# send transactions
transactions = self.transaction_queue.peek()
if transactions:
log.debug("sending transactions", remote_id=proto)
proto.send_transactions(*transactions)
else:
log.debug("peer failed to answer DAO challenge, stop.",
proto=proto)
if proto.peer:
proto.peer.stop()
del self.dao_challenges[proto]
# transactions
def on_receive_transactions(self, proto, transactions):
"receives rlp.decoded serialized"
log.debug('----------------------------------')
log.debug('remote_transactions_received',
count=len(transactions),
remote_id=proto)
for tx in transactions:
self.add_transaction(tx, origin=proto)
# blockhashes ###########
def on_newblockhashes(self, proto, newblockhashes):
"""
msg sent out if not the full block is propagated
chances are high, that we get the newblock, though.
"""
log.debug('----------------------------------')
log.debug("recv newblockhashes",
num=len(newblockhashes),
remote_id=proto)
assert len(newblockhashes) <= 256
self.synchronizer.receive_newblockhashes(proto, newblockhashes)
def on_receive_getblockheaders(self, proto, hash_or_number, block, amount,
skip, reverse):
hash_mode = 1 if hash_or_number[0] else 0
block_id = encode_hex(
hash_or_number[0]) if hash_mode else hash_or_number[1]
log.debug('----------------------------------')
log.debug("handle_getblockheaders", amount=amount, block=block_id)
headers = []
max_hashes = min(amount, self.wire_protocol.max_getblockheaders_count)
if hash_mode:
origin_hash = hash_or_number[0]
else:
if is_dao_challenge(self.config['eth']['block'], hash_or_number[1],
amount, skip):
log.debug("sending: answer DAO challenge")
headers.append(build_dao_header(self.config['eth']['block']))
proto.send_blockheaders(*headers)
return
try:
origin_hash = self.chain.get_blockhash_by_number(
hash_or_number[1])
except KeyError:
origin_hash = b''
if not origin_hash or not self.chain.has_blockhash(origin_hash):
log.debug('unknown block: {}'.format(encode_hex(origin_hash)))
proto.send_blockheaders(*[])
return
headers = self.query_headers(
hash_mode,
max_hashes,
skip,
reverse,
origin_hash=origin_hash,
number=block_id,
)
log.debug("sending: found blockheaders", count=len(headers))
proto.send_blockheaders(*headers)
def on_receive_blockheaders(self, proto, blockheaders):
log.debug('----------------------------------')
if blockheaders:
log.debug("on_receive_blockheaders",
count=len(blockheaders),
remote_id=proto,
first=encode_hex(blockheaders[0].hash),
last=encode_hex(blockheaders[-1].hash))
else:
log.debug("recv 0 remote block headers, signifying genesis block")
if proto in self.dao_challenges:
self.dao_challenges[proto][0].receive_blockheaders(
proto, blockheaders)
else:
self.synchronizer.receive_blockheaders(proto, blockheaders)
# blocks ################
def on_receive_getblockbodies(self, proto, blockhashes):
log.debug('----------------------------------')
log.debug("on_receive_getblockbodies", count=len(blockhashes))
found = []
for bh in blockhashes[:self.wire_protocol.max_getblocks_count]:
try:
found.append(self.chain.get_block(bh))
except KeyError:
log.debug("unknown block requested", block_hash=encode_hex(bh))
if found:
log.debug("found", count=len(found))
proto.send_blockbodies(*found)
def on_receive_blockbodies(self, proto, bodies):
log.debug('----------------------------------')
log.debug("recv block bodies", count=len(bodies), remote_id=proto)
if bodies:
self.synchronizer.receive_blockbodies(proto, bodies)
def on_receive_newblock(self, proto, block, chain_difficulty):
log.debug('----------------------------------')
log.debug("recv newblock", block=block, remote_id=proto)
self.synchronizer.receive_newblock(proto, block, chain_difficulty)