class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote
electrum servers, each connection is handled by its own
thread object returned from Interface(). Its external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None, active_chain=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
if active_chain is None:
active_chain = chainparams.get_active_chain()
self.active_chain = active_chain
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self, self.active_chain)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# run() will block while this event is cleared
self.chain_switched = threading.Event()
self.chain_switched.set()
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.sanitize_default_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def switch_chains(self, chaincode=None):
if chaincode is None:
chain = chainparams.get_active_chain()
else:
chain = chainparams.get_chain_instance(chaincode)
self.chain_switched.clear()
self.active_chain = chain
if self.config.get_active_chain_code() != self.active_chain.code:
self.config.set_active_chain_code(self.active_chain.code)
self.print_error('switching chains to {}'.format(chain.code))
self.stop_network()
time.sleep(0.2)
self.bc_requests.clear()
self.blockchain = Blockchain(self.config, self, self.active_chain)
self.queue = Queue.Queue()
self.sanitize_default_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.interface = None
self.interfaces = {}
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
self.blockchain.init()
# Start the new network
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
time.sleep(0.2)
self.chain_switched.set()
def sanitize_default_server(self):
"""Load the default server from config."""
self.default_server = self.config.get('server')
self.use_ssl = self.config.get('use_ssl', True)
try:
deserialize_server(self.default_server)
except Exception:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server(active_chain=self.active_chain, protocol=get_protocol_letter(self.use_ssl))
def read_recent_servers(self):
return self.config.get('recent_servers', [])
def save_recent_servers(self):
self.config.set_key('recent_servers', self.recent_servers, True)
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
for r in self.unanswered_requests.values():
self.interface.send_request(r)
for addr in self.subscribed_addresses:
self.interface.send_request({'method':'blockchain.address.subscribe','params':[addr]})
self.interface.send_request({'method':'server.banner','params':[]})
self.interface.send_request({'method':'server.peers.subscribe','params':[]})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect()
def auto_connect(self):
return self.config.get('auto_connect', True)
def get_interfaces(self):
'''The interfaces that are in connected state'''
return [s for s, i in self.interfaces.items() if i.is_connected()]
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = self.active_chain.DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if not server in self.interfaces.keys():
if server == self.default_server:
self.set_status('connecting')
i = interface.Interface(server, self.queue, self.config)
self.interfaces[i.server] = i
i.start()
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for i in self.interfaces.values():
i.stop()
self.interface = None
self.interfaces = {}
def set_parameters(self, host, port, protocol, proxy, auto_connect):
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
servers = self.get_interfaces() # Those in connected state
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion = None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect():
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.print_error("starting %s; will switch once connected" % server)
self.start_interface(server)
return
i = self.interfaces[server]
if not i.is_connected():
# do nothing; we will switch once connected
return
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.stop_interface()
self.interface = i
self.addr_responses = {}
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
if self.interface:
self.interface.stop()
self.interface = None
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_if_notification(self, i):
'''Handle interface addition and removal through notifications'''
if i.is_connected():
self.add_recent_server(i)
i.send_request({'method':'blockchain.headers.subscribe','params':[]})
if i.server == self.default_server:
self.switch_to_interface(i.server)
else:
self.interfaces.pop(i.server, None)
self.heights.pop(i.server, None)
if i == self.interface:
self.interface = None
self.addr_responses = {}
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
# Our set of interfaces changed
self.notify('interfaces')
def process_response(self, i, response):
# the id comes from the daemon or the network proxy
_id = response.get('id')
if _id is not None:
if i != self.interface:
return
self.unanswered_requests.pop(_id)
method = response.get('method')
result = response.get('result')
if method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
self.banner = result
self.notify('banner')
elif method == 'blockchain.address.subscribe':
addr = response.get('params')[0]
self.addr_responses[addr] = result
self.response_queue.put(response)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(i, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(i, response)
else:
self.response_queue.put(response)
def handle_requests(self):
'''Some requests require connectivity, others we handle locally in
process_request() and must do so in order to e.g. prevent the
daemon seeming unresponsive.
'''
unhandled = []
while not self.requests_queue.empty():
request = self.requests_queue.get()
if not self.process_request(request):
unhandled.append(request)
for request in unhandled:
self.requests_queue.put(request)
def process_request(self, request):
'''Returns true if the request was processed.'''
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
out['result'] = f(*params)
except AttributeError:
out['error'] = "unknown method"
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return True
if method == 'blockchain.address.subscribe':
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({'id':_id, 'result':self.addr_responses[addr]})
return True
# This request needs connectivity. If we don't have an
# interface, we cannot process it.
if not self.is_connected():
return False
self.unanswered_requests[_id] = request
self.interface.send_request(request)
return True
def check_interfaces(self):
now = time.time()
# nodes
if len(self.interfaces) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('retrying connections to reach preferred number of interfaces ({}/{})'.format(len(self.interfaces), self.num_server))
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
interface.send_request({'method':'blockchain.block.get_chunk', 'params':[idx]})
data['chunk_idx'] = idx
data['req_time'] = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx, response['result'])
# If not finished, get the next chunk
if idx < 0 or not idx or self.get_local_height() >= data['if_height']:
self.bc_requests.popleft()
if not idx:
interface.print_error("header didn't match checkpoint, dismissing interface")
interface.stop()
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
interface.send_request({'method':'blockchain.block.get_header', 'params':[height]})
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(data['chain'], response['result'])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify('updated')
else:
interface.print_error("header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them, if necessary'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) / self.active_chain.chunk_size)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface.is_connected():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
interface.stop()
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def run(self):
self.blockchain.init()
while self.is_running():
# wait if we're switching chains
self.chain_switched.wait()
self.check_interfaces()
self.handle_requests()
self.handle_bc_requests()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
# if response is None it is a notification about the interface
if response is None:
self.process_if_notification(i)
else:
self.process_response(i, response)
self.stop_network()
self.print_error("stopped")
def on_header(self, i, r):
header = r.get('result')
if not header:
return
height = header.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get('merkle_root')
self.utxo_roots[i.server] = header.get('utxo_root')
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {'if_height': height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote
electrum servers, each connection is handled by its own
thread object returned from Interface(). Its external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None, active_chain=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
if active_chain is None:
active_chain = chainparams.get_active_chain()
self.active_chain = active_chain
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self, self.active_chain)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# run() will block while this event is cleared
self.chain_switched = threading.Event()
self.chain_switched.set()
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.sanitize_default_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.start_network(
deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def switch_chains(self, chaincode=None):
if chaincode is None:
chain = chainparams.get_active_chain()
else:
chain = chainparams.get_chain_instance(chaincode)
self.chain_switched.clear()
self.active_chain = chain
if self.config.get_active_chain_code() != self.active_chain.code:
self.config.set_active_chain_code(self.active_chain.code)
self.print_error('switching chains to {}'.format(chain.code))
self.stop_network()
time.sleep(0.2)
self.bc_requests.clear()
self.blockchain = Blockchain(self.config, self, self.active_chain)
self.queue = Queue.Queue()
self.sanitize_default_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.interface = None
self.interfaces = {}
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
self.blockchain.init()
# Start the new network
self.start_network(
deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
time.sleep(0.2)
self.chain_switched.set()
def sanitize_default_server(self):
"""Load the default server from config."""
self.default_server = self.config.get('server')
self.use_ssl = self.config.get('use_ssl', True)
try:
deserialize_server(self.default_server)
except Exception:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server(
active_chain=self.active_chain,
protocol=get_protocol_letter(self.use_ssl))
def read_recent_servers(self):
return self.config.get('recent_servers', [])
def save_recent_servers(self):
self.config.set_key('recent_servers', self.recent_servers, True)
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' %
(self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error('sending subscriptions to', self.interface.server,
len(self.unanswered_requests),
len(self.subscribed_addresses))
for r in self.unanswered_requests.values():
self.interface.send_request(r)
for addr in self.subscribed_addresses:
self.interface.send_request({
'method': 'blockchain.address.subscribe',
'params': [addr]
})
self.interface.send_request({'method': 'server.banner', 'params': []})
self.interface.send_request({
'method': 'server.peers.subscribe',
'params': []
})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({
'method': 'network.status',
'params': [key, value]
})
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect()
def auto_connect(self):
return self.config.get('auto_connect', True)
def get_interfaces(self):
'''The interfaces that are in connected state'''
return [s for s, i in self.interfaces.items() if i.is_connected()]
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = self.active_chain.DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if not server in self.interfaces.keys():
if server == self.default_server:
self.set_status('connecting')
i = interface.Interface(server, self.queue, self.config)
self.interfaces[i.server] = i
i.start()
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol,
exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"],
int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(
socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for i in self.interfaces.values():
i.stop()
self.interface = None
self.interfaces = {}
def set_parameters(self, host, port, protocol, proxy, auto_connect):
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
servers = self.get_interfaces() # Those in connected state
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion=None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect():
if suggestion and self.protocol == deserialize_server(
suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.print_error("starting %s; will switch once connected" %
server)
self.start_interface(server)
return
i = self.interfaces[server]
if not i.is_connected():
# do nothing; we will switch once connected
return
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.stop_interface()
self.interface = i
self.addr_responses = {}
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
if self.interface:
self.interface.stop()
self.interface = None
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0, s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_if_notification(self, i):
'''Handle interface addition and removal through notifications'''
if i.is_connected():
self.add_recent_server(i)
i.send_request({
'method': 'blockchain.headers.subscribe',
'params': []
})
if i.server == self.default_server:
self.switch_to_interface(i.server)
else:
self.interfaces.pop(i.server, None)
self.heights.pop(i.server, None)
if i == self.interface:
self.interface = None
self.addr_responses = {}
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
# Our set of interfaces changed
self.notify('interfaces')
def process_response(self, i, response):
# the id comes from the daemon or the network proxy
_id = response.get('id')
if _id is not None:
if i != self.interface:
return
self.unanswered_requests.pop(_id)
method = response.get('method')
result = response.get('result')
if method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
self.banner = result
self.notify('banner')
elif method == 'blockchain.address.subscribe':
addr = response.get('params')[0]
self.addr_responses[addr] = result
self.response_queue.put(response)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(i, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(i, response)
else:
self.response_queue.put(response)
def handle_requests(self):
'''Some requests require connectivity, others we handle locally in
process_request() and must do so in order to e.g. prevent the
daemon seeming unresponsive.
'''
unhandled = []
while not self.requests_queue.empty():
request = self.requests_queue.get()
if not self.process_request(request):
unhandled.append(request)
for request in unhandled:
self.requests_queue.put(request)
def process_request(self, request):
'''Returns true if the request was processed.'''
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id': _id}
try:
f = getattr(self, method[8:])
out['result'] = f(*params)
except AttributeError:
out['error'] = "unknown method"
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return True
if method == 'blockchain.address.subscribe':
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({
'id': _id,
'result': self.addr_responses[addr]
})
return True
# This request needs connectivity. If we don't have an
# interface, we cannot process it.
if not self.is_connected():
return False
self.unanswered_requests[_id] = request
self.interface.send_request(request)
return True
def check_interfaces(self):
now = time.time()
# nodes
if len(self.interfaces) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error(
'retrying connections to reach preferred number of interfaces ({}/{})'
.format(len(self.interfaces), self.num_server))
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
interface.send_request({
'method': 'blockchain.block.get_chunk',
'params': [idx]
})
data['chunk_idx'] = idx
data['req_time'] = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx,
response['result'])
# If not finished, get the next chunk
if idx < 0 or not idx or self.get_local_height(
) >= data['if_height']:
self.bc_requests.popleft()
if not idx:
interface.print_error(
"header didn't match checkpoint, dismissing interface"
)
interface.stop()
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
interface.send_request({
'method': 'blockchain.block.get_header',
'params': [height]
})
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(
data['chain'], response['result'])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify('updated')
else:
interface.print_error(
"header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them, if necessary'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) /
self.active_chain.chunk_size)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface.is_connected():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
interface.stop()
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def run(self):
self.blockchain.init()
while self.is_running():
# wait if we're switching chains
self.chain_switched.wait()
self.check_interfaces()
self.handle_requests()
self.handle_bc_requests()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
# if response is None it is a notification about the interface
if response is None:
self.process_if_notification(i)
else:
self.process_response(i, response)
self.stop_network()
self.print_error("stopped")
def on_header(self, i, r):
header = r.get('result')
if not header:
return
height = header.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get('merkle_root')
self.utxo_roots[i.server] = header.get('utxo_root')
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {'if_height': height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
