def request_state_components(self):
"""
Ask all our peers for any StateComponents that our Blockchain needs, and
that we have not recently requested.
"""
with self.lock:
if not self.blockchain.state_available:
# We need to ask for StateComponents from our peers until we get
# all of them.
for requested_component in \
self.blockchain.get_requested_state_components():
if (requested_component
in self.recently_requested_state_components):
# We just asked for this one
continue
logging.debug("Requesting StateComponent {} from all "
"peers".format(
bytes2string(requested_component)))
# Compose just one message per request
message = ["GETSTATE", bytes2string(requested_component)]
for connection in self.connections:
# Send it to all peers. Hopefully one or more of them
# will send us a StateComponent back.
connection.send_message(message)
# Remember having asked for this component
self.recently_requested_state_components.add(
requested_component)
def announce_block(self, block_hash):
"""
Tell all of our connected peers about a block we have.
"""
with self.lock:
for connection in self.connections:
# Send an INV message with the hash of the thing we have, in
# case they want it.
connection.send_message(["INV", bytes2string(block_hash)])
def announce_transaction(self, transaction_hash):
"""
Tell all of our connected peers about a transaction we have.
"""
with self.lock:
for connection in self.connections:
# Send a TXINV message with the hash of the thing we have, in
# case they want it.
connection.send_message(
["TXINV", bytes2string(transaction_hash)])
def process_inv_queue(self):
"""
Download some blocks from the queue of blocks we know we need.
"""
for _ in xrange(100):
# Only ask for 100 blocks at a time.
if len(self.inv_queue) > 0:
# Get the bytestring hash of the next block to ask for
block_hash = self.inv_queue.popleft()
self.inv_queued.remove(block_hash)
if (self.factory.peer.blockchain.needs_block(block_hash) and
block_hash not in self.block_queued):
# We need this block and don't have it in our queue to add
# to the blockchain, so get it. TODO: Check to see if we
# requested it recently.
self.send_message(["GETDATA", bytes2string(block_hash)])
# Process the inv_queue again.
self.inv_watcher = reactor.callLater(1, self.process_inv_queue)
def tick(self):
"""
See if we have the optimal number of outgoing connections. If we have
too few, add some.
"""
with self.lock:
# How many connections do we have right now?
current_connections = len(self.connections)
logging.info("Tick {} from localhost port {}: {} of {} "
"connections".format(self.tick_count, self.port,
current_connections,
self.optimal_connections))
for connection in self.connections:
logging.info(
"\tTo {} port {}".format(*connection.remote_address))
logging.info("{} outgoing connections:".format(
len(self.outgoing_hosts)))
for host in self.outgoing_hosts:
logging.info("\t To {}".format(host))
logging.info("Blockchain height: {}".format(
len(self.blockchain.hashes_by_height)))
logging.info("Blocks known: {}".format(
self.blockchain.get_block_count()))
logging.info("Blocks pending: {}".format(
len(self.blockchain.pending_callbacks)))
logging.info("State available for verification and mining:"
" {}".format(self.blockchain.state_available))
logging.info("Transactions pending: {}".format(
len(self.blockchain.transactions)))
# Calculate the average block time in seconds for the last few
# blocks. It may be None, meaning we don't have enough history to
# work it out.
block_time = self.blockchain.get_average_block_time()
if block_time is not None:
logging.info(
"Average block time: {} seconds".format(block_time))
logging.info("Blockchain disk usage: {} bytes".format(
self.blockchain.get_disk_usage()))
if self.tick_count % 60 == 0:
# Also log some science stats every 60 ticks (by default, 1 hour)
pybc.science.log_event("chain_height",
len(self.blockchain.hashes_by_height))
pybc.science.log_event("pending_transactions",
len(self.blockchain.transactions))
pybc.science.log_event("connections", current_connections)
if block_time is not None:
# We have an average block time, so record that.
pybc.science.log_event("block_time", block_time)
# We want to log how much space our databases (block store and
# state) take.
pybc.science.log_event("blockchain_usage",
self.blockchain.get_disk_usage())
pybc.science.log_filesize("blockchain_file_usage",
self.blockchain.store_filename)
# Request whatever state components we need, egardless if whether we
# recently requested them, in case our download got stalled
self.recently_requested_state_components = set()
self.request_state_components()
if (len(self.outgoing_hosts) < self.optimal_connections
and len(self.known_peers) > 0):
# We don't have enough outgoing connections, but we do know some
# peers.
for i in range(
min(
self.connections_per_batch,
self.optimal_connections -
len(self.outgoing_hosts))):
# Try several connections in a batch.
for _ in range(self.optimal_connections -
len(self.outgoing_hosts)):
# For each connection we want but don't have
# Find a peer we aren't connected to and connect to them
key = random.sample(self.known_peers, 1)[0]
if key.count(':'):
host = key.split(':')[0]
else:
host = key
# Try at most 100 times to find a host we aren't
# connected to, and which isn't trivially obviously us.
attempt = 1
while (host in self.outgoing_hosts
or host in self.incoming_hosts or host
== self.external_address) and attempt < 100:
# Try a new host
key = random.sample(self.known_peers, 1)[0]
if key.count(':'):
host = key.split(':')[0]
else:
host = key
# Increment attempt
attempt += 1
# TODO: This always makes two attempts at least
if attempt < 100:
# We found one!
# Connect to it.
# Get the port (and discard the last heard from
# time)
port, _ = self.known_peers[key]
# Connect to it.
self.connect(host, port)
else:
# No more things we can try connecting to
break
# Throw out peers that are too old. First compile a list of their
# hostnames.
too_old = []
for key, (port, last_seen) in six.iteritems(self.known_peers):
if last_seen is None:
# This is a bootstrap peer. Don't remove it.
continue
if time.time() - last_seen > self.peer_timeout:
# We haven't heard from/about this node recently enough.
too_old.append(key)
# Now drop all the too old hosts
for key in too_old:
del self.known_peers[key]
# Broadcast all our hosts.
logging.info("{} known peers".format(len(self.known_peers)))
for key, (port, last_seen) in six.iteritems(self.known_peers):
if last_seen is None:
# This is a bootstrap peer, so don't announce it.
continue
if key.count(':'):
host, _ = key.split(':')
logging.debug("\tPeer {} port {} last seen {}".format(
host, port, time2string(last_seen)))
for connection in self.connections:
connection.send_message(["ADDR", host, port, last_seen])
if self.external_address is not None:
# Broadcast ourselves, since we know our address.
for connection in self.connections:
connection.send_message([
"ADDR", self.external_address, self.port,
int(time.time())
])
# Do we need to re-poll for blocks?
if self.repoll:
self.repoll = False
logging.warning("Repolling due to unverifiable block.")
# Compose just one message
message = (["GETBLOCKS"] + [
bytes2string(block_hash)
for block_hash in self.blockchain.get_block_locator()
])
for connection in self.connections:
connection.send_message(message)
logging.info("Saving...")
# Keep track of how long this takes.
save_start = time.clock()
if self.tick_count % 60 == 0:
pybc.science.start_timer("sync")
# Sync the blockchain to disk. This needs to lock the blockchain, so
# the blockchain must never wait on the peer while holding its lock.
# Hence the complicated deferred callback system. TODO: replace the
# whole business with events.
self.blockchain.sync()
# Sync the known peers to disk
self.known_peers.sync()
if self.tick_count % 60 == 0:
pybc.science.stop_timer("sync")
# How long did it take?
save_end = time.clock()
logging.info("Saved to disk in {:.2} seconds".format(save_end -
save_start))
# Count this tick as having happened
self.tick_count += 1
# Tick again later
reactor.callLater(self.tick_period, self.tick)
logging.info("Tick complete.")
def handle_message(self, parts):
"""
Given a message as a list of string parts, handle it. Meant to run non-
blocking in its own thread.
Schedules any reply and any actions to be taken to be done in the main
Twisted thread.
"""
try:
if len(parts) == 0:
# Skip empty lines
return
if parts[0] == "NETWORK":
# This is a network command, telling us the network and version
# of the remote host. If we like them (i.e. they are equal to
# ours), send back an acknowledgement with our network info.
# Also start requesting peers and blocks from them.
if (parts[1] == self.factory.peer.network and
int(parts[2]) == self.factory.peer.version):
# We like their network and version number.
# Send back a network OK command with our own.
self.send_message(["NETWORK-OK", self.factory.peer.network,
self.factory.peer.version])
# Ask them for peers
self.send_message(["GETADDR"])
# Ask them for the blocks we need, given our list of block
# locator hashes.
self.send_message(["GETBLOCKS"] +
[bytes2string(block_hash) for block_hash in
self.factory.peer.blockchain.get_block_locator()])
# Send all the pending transactions
for transaction_hash, _ in \
self.factory.peer.blockchain.get_transactions():
self.send_message(["TXINV", bytes2string(
transaction_hash)])
else:
# Nope, we don't like them.
# Disconnect
self.disconnect()
elif parts[0] == "NETWORK-OK":
# This is a network OK command, telling us that they want to
# talk to us, and giving us their network and version number. If
# we like their network and version number too, we can start
# exchanging peer info.
if (parts[1] == self.factory.peer.network and
int(parts[2]) == self.factory.peer.version):
# We like their network and version number. Send them a
# getaddr message requesting a list of peers. The next thing
# they give us might be something completely different, but
# that's OK; they ought to send some peers eventually.
self.send_message(["GETADDR"])
# Ask them for the blocks we need, given our list of block
# locator hashes.
self.send_message(["GETBLOCKS"] +
[bytes2string(block_hash) for block_hash in
self.factory.peer.blockchain.get_block_locator()])
# Send all the pending transactions
for transaction_hash, _ in \
self.factory.peer.blockchain.get_transactions():
self.send_message(["TXINV", bytes2string(
transaction_hash)])
else:
# We don't like their network and version. Drop them.
# Disconnect
self.disconnect()
elif parts[0] == "GETADDR":
# They want a list of all known peers.
# Send them ADDR messages, one per known peer.
for host, port, time_seen in self.factory.peer.get_peers():
if time_seen is None:
# This is a bootstrap peer
continue
# Send the peer's host and port in an ADDR message
self.send_message(["ADDR", host, port, time_seen])
elif parts[0] == "ADDR":
# They claim that there is a peer. Tell our peer the host and port
# and time seen.
self.factory.peer.peer_seen(parts[1], int(parts[2]),
int(parts[3]))
elif parts[0] == "GETBLOCKS":
# They gave us a block locator. Work out the blocks they need,
# and send INV messages about them.
# Decode all the hex hashes to bytestring
block_hashes = [string2bytes(part) for part in parts[1:]]
for needed_hash in \
self.factory.peer.blockchain.blocks_after_locator(
block_hashes):
# They need this hash. Send an INV message about it.
# TODO: consolidate and limit these.
self.send_message(["INV", bytes2string(needed_hash)])
elif parts[0] == "INV":
# They have a block. If we don't have it, ask for it.
# TODO: allow advertising multiple things at once.
# Decode the hash they have
block_hash = string2bytes(parts[1])
if (self.factory.peer.blockchain.needs_block(block_hash) and
block_hash not in self.inv_queued and
block_hash not in self.block_queued):
# We need this block, and it isn't in any queues.
self.inv_queue.append(block_hash)
self.inv_queued.add(block_hash)
elif parts[0] == "GETDATA":
# They want the data for a block. Send it to them if we have
# it.
# Decode the hash they want
block_hash = string2bytes(parts[1])
if self.factory.peer.blockchain.has_block(block_hash):
# Get the block to send
block = self.factory.peer.blockchain.get_block(block_hash)
# Send them the block. TODO: This encoding is terribly
# inefficient, but we can't send it as binary without them
# switching out of line mode, and they don't know to do that
# because messages queue.
self.send_message(["BLOCK", bytes2string(block.to_bytes())])
else:
logging.error("Can't send missing block: '{}'".format(
bytes2string(block_hash)))
elif parts[0] == "BLOCK":
# They have sent us a block. Add it if it is valid.
# Decode the block bytes
block_bytes = string2bytes(parts[1])
# Make a Block object
block = Block.from_bytes(block_bytes)
if block.block_hash() not in self.block_queued:
# Queue it if it's not already queued. Because of the set it
# can only be queued once.
self.block_queue.append(block)
self.block_queued.add(block.block_hash())
elif parts[0] == "TXINV":
# They have sent us a hash of a transaction that they have. If
# we don't have it, we should get it and pass it on.
# Decode the hash they have
transaction_hash = string2bytes(parts[1])
if not self.factory.peer.blockchain.has_transaction(
transaction_hash):
# We need this transaction!
self.send_message(["GETTX", parts[1]])
elif parts[0] == "GETTX":
# They want a transaction from our blockchain.
# Decode the hash they want
transaction_hash = string2bytes(parts[1])
if self.factory.peer.blockchain.has_transaction(transaction_hash):
# Get the transaction to send
transaction = self.factory.peer.blockchain.get_transaction(
transaction_hash)
if transaction is not None:
# We have it (still). Send them the block transaction.
self.send_message(["TX", bytes2string(transaction)])
else:
logging.error("Lost transaction: '{}'".format(
bytes2string(transaction_hash)))
else:
logging.error("Can't send missing transaction: '{}'".format(
bytes2string(transaction_hash)))
elif parts[0] == "TX":
# They have sent us a transaction. Add it to our blockchain.
# Decode the transaction bytes
transaction_bytes = string2bytes(parts[1])
logging.debug("Incoming transaction.")
if self.factory.peer.blockchain.transaction_valid_for_relay(
transaction_bytes):
# This is a legal transaction to accept from a peer (not
# something like a block reward).
logging.debug("Transaction acceptable from peer.")
# Give it to the blockchain as bytes. The blockchain can
# determine whether to forward it on or not and call the
# callback with transaction hash and transaction status
# (True or False).
self.factory.peer.send_transaction(
transaction_bytes)
elif parts[0] == "GETSTATE":
# They're asking us for a StateComponent from our State, with
# the given hash. We just serve these like a dumb server.
# We assume this is under our current State. If it's not, we'll
# return a nort found message and make them start over again
# from the top.
# What StateComponent do they want?
state_hash = string2bytes(parts[1])
# Go get it
state_component = \
self.factory.peer.blockchain.get_state_component(state_hash)
if state_component is None:
# Complain we don't have that. They need to start over from
# the state hash in the latest block.
logging.warning("Peer requested nonexistent StateComponent "
"{} vs. root hash {}".format(parts[1],
bytes2string(
self.factory.peer.blockchain.get_state_hash())))
self.send_message(["NOSTATE", bytes2string(state_hash)])
else:
logging.debug("Fulfilling request for StateComponent "
"{}".format(parts[1]))
# Pack up the StateComponent's data as bytes and send it
# along. They'll know which one it was when they hash it.
self.send_message(["STATE",
bytes2string(state_component.data)])
elif parts[0] == "NOSTATE":
# They're saying they don't have a StateComponent we asked for.
logging.warning("Peer says they do not have StateComponent: "
"{}".format(parts[1]))
elif parts[0] == "STATE":
# They're sending us a StateComponent we probably asked for.
# Unpack the data
component_bytestring = string2bytes(parts[1])
# Give it to the Blockchain. It knows how to handle these
# things. TODO: Maybe put a queue here, since this potentially
# does the whole state rebuilding operation.
self.factory.peer.blockchain.add_state_component(
component_bytestring)
# Tell the peer to request more StateComponents.
# TODO: This is going to blow up.
self.factory.peer.request_state_components()
elif parts[0] == "ERROR":
# The remote peer didn't like something.
# Print debugging output.
logging.error("Error from remote peer: {}".format(" ".join(
parts[1:])))
else:
# They're trying to send a command we don't know about.
# Complain.
logging.error("Remote host tried unknown command {}".format(
parts[1]))
self.send_message(["ERROR", parts[0]])
if not self.incoming:
# We processed a valid message from a peer we connected out to.
# Record that we've seen them for anouncement purposes.
self.factory.peer.peer_seen(self.remote_address[0],
self.remote_address[1], int(time.time()))
except BaseException:
logging.error("Exception processing command: {}".format(parts))
logging.error(traceback.format_exc())
# Disconnect from people who send us garbage
self.disconnect()