def run_cmdline(self, config_options):
password = config_options.get('password')
new_password = config_options.get('new_password')
config = SimpleConfig(config_options)
config.fee_estimates = self.network.config.fee_estimates.copy()
cmdname = config.get('cmd')
cmd = known_commands[cmdname]
if cmd.requires_wallet:
path = config.get_wallet_path()
wallet = self.wallets.get(path)
if wallet is None:
return {
'error':
'Wallet not open. Use "electrum-vtc daemon load_wallet"'
}
else:
wallet = None
# arguments passed to function
args = map(lambda x: config.get(x), cmd.params)
# decode json arguments
args = map(json_decode, args)
# options
args += map(
lambda x:
(config_options.get(x)
if x in ['password', 'new_password'] else config.get(x)),
cmd.options)
cmd_runner = Commands(config, wallet, self.network)
func = getattr(cmd_runner, cmd.name)
result = func(*args)
return result
def run_cmdline(self, config_options):
password = config_options.get('password')
config = SimpleConfig(config_options)
cmdname = config.get('cmd')
cmd = known_commands[cmdname]
wallet = self.load_wallet(config) if cmd.requires_wallet else None
# arguments passed to function
args = map(lambda x: config.get(x), cmd.params)
# decode json arguments
args = map(json_decode, args)
# options
args += map(lambda x: config.get(x), cmd.options)
cmd_runner = Commands(config, wallet, self.network)
cmd_runner.password = password
func = getattr(cmd_runner, cmd.name)
result = func(*args)
return result
def run_cmdline(self, config_options):
config = SimpleConfig(config_options)
cmdname = config.get('cmd')
cmd = known_commands[cmdname]
wallet = self.load_wallet(config) if cmd.requires_wallet else None
# arguments passed to function
args = map(lambda x: config.get(x), cmd.params)
# decode json arguments
args = map(json_decode, args)
# options
args += map(lambda x: config.get(x), cmd.options)
cmd_runner = Commands(config, wallet, self.network,
password=config_options.get('password'),
new_password=config_options.get('new_password'))
func = getattr(cmd_runner, cmd.name)
result = func(*args)
return result
def run_cmdline(self, config_options):
password = config_options.get("password")
config = SimpleConfig(config_options)
cmdname = config.get("cmd")
cmd = known_commands[cmdname]
wallet = self.load_wallet(config) if cmd.requires_wallet else None
if wallet:
wallet.wait_until_synchronized()
# arguments passed to function
args = map(lambda x: config.get(x), cmd.params)
# decode json arguments
args = map(json_decode, args)
# options
args += map(lambda x: config.get(x), cmd.options)
cmd_runner = Commands(config, wallet, self.network)
cmd_runner.password = password
func = getattr(cmd_runner, cmd.name)
result = func(*args)
return result
def run_daemon(self, config_options):
config = SimpleConfig(config_options)
sub = config.get('subcommand')
assert sub in [
None, 'start', 'stop', 'status', 'load_wallet', 'close_wallet'
]
if sub in [None, 'start']:
response = "Daemon already running"
elif sub == 'load_wallet':
path = config.get_wallet_path()
wallet = self.load_wallet(path, config.get('password'))
self.cmd_runner.wallet = wallet
response = True
elif sub == 'close_wallet':
path = config.get_wallet_path()
if path in self.wallets:
self.stop_wallet(path)
response = True
else:
response = False
elif sub == 'status':
if self.network:
p = self.network.get_parameters()
response = {
'path': self.network.config.path,
'server': p[0],
'blockchain_height': self.network.get_local_height(),
'server_height': self.network.get_server_height(),
'spv_nodes': len(self.network.get_interfaces()),
'connected': self.network.is_connected(),
'auto_connect': p[4],
'version': ELECTRUM_VERSION,
'wallets':
{k: w.is_up_to_date()
for k, w in self.wallets.items()},
'fee_per_kb': self.config.fee_per_kb(),
}
else:
response = "Daemon offline"
elif sub == 'stop':
self.stop()
response = "Daemon stopped"
return response
def run_gui(self, config_options):
config = SimpleConfig(config_options)
if self.gui:
if hasattr(self.gui, 'new_window'):
path = config.get_wallet_path()
self.gui.new_window(path, config.get('url'))
response = "ok"
else:
response = "error: current GUI does not support multiple windows"
else:
response = "Error: Electrum is running in daemon mode. Please stop the daemon first."
return response
def run_gui(self, config_options):
config = SimpleConfig(config_options)
if self.gui:
if hasattr(self.gui, 'new_window'):
path = config.get_wallet_path()
self.gui.new_window(path, config.get('url'))
response = "ok"
else:
response = "error: current GUI does not support multiple windows"
else:
response = "Error: Electrum is running in daemon mode. Please stop the daemon first."
return response
def run_daemon(self, config_options):
config = SimpleConfig(config_options)
sub = config.get('subcommand')
assert sub in [None, 'start', 'stop', 'status', 'load_wallet', 'close_wallet']
if sub in [None, 'start']:
response = "Daemon already running"
elif sub == 'load_wallet':
path = config.get_wallet_path()
wallet = self.load_wallet(path, config.get('password'))
self.cmd_runner.wallet = wallet
response = True
elif sub == 'close_wallet':
path = config.get_wallet_path()
if path in self.wallets:
self.stop_wallet(path)
response = True
else:
response = False
elif sub == 'status':
if self.network:
p = self.network.get_parameters()
response = {
'path': self.network.config.path,
'server': p[0],
'blockchain_height': self.network.get_local_height(),
'server_height': self.network.get_server_height(),
'spv_nodes': len(self.network.get_interfaces()),
'connected': self.network.is_connected(),
'auto_connect': p[4],
'version': ELECTRUM_VERSION,
'wallets': {k: w.is_up_to_date()
for k, w in self.wallets.items()},
'fee_per_kb': self.config.fee_per_kb(),
}
else:
response = "Daemon offline"
elif sub == 'stop':
self.stop()
response = "Daemon stopped"
return response
def run_cmdline(self, config_options):
password = config_options.get('password')
new_password = config_options.get('new_password')
config = SimpleConfig(config_options)
config.fee_estimates = self.network.config.fee_estimates.copy()
cmdname = config.get('cmd')
cmd = known_commands[cmdname]
if cmd.requires_wallet:
path = config.get_wallet_path()
wallet = self.wallets.get(path)
if wallet is None:
return {'error': 'Wallet not open. Use "electrum daemon load_wallet"'}
else:
wallet = None
# arguments passed to function
args = map(lambda x: config.get(x), cmd.params)
# decode json arguments
args = map(json_decode, args)
# options
args += map(lambda x: config.get(x), cmd.options)
cmd_runner = Commands(config, wallet, self.network, password=password, new_password=new_password)
func = getattr(cmd_runner, cmd.name)
result = func(*args)
return result
class Network(threading.Thread):
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.protocol = self.config.get('protocol','s')
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.disconnected_time = time.time()
self.recent_servers = self.config.get('recent_servers',[]) # successful connections
self.pending_servers = set()
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# address subscriptions and cached results
self.addresses = {}
self.connection_status = 'connecting'
self.requests_queue = Queue.Queue()
def get_server_height(self):
return self.heights.get(self.default_server,0)
def server_is_lagging(self):
h = self.get_server_height()
if not h:
print_error('no height for main interface')
return False
lag = self.get_local_height() - self.get_server_height()
return lag > 1
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected
def send_subscriptions(self):
for addr in self.addresses:
self.interface.send_request({'method':'blockchain.address.subscribe', 'params':[addr]})
self.interface.send_request({'method':'server.banner','params':[]})
self.interface.send_request({'method':'server.peers.subscribe','params':[]})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list:
return
server = random.choice( choice_list )
return server
def get_parameters(self):
host, port, protocol = self.default_server.split(':')
proxy = self.proxy
auto_connect = self.config.get('auto_cycle', True)
return host, port, protocol, proxy, auto_connect
def get_interfaces(self):
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.pending_servers.add(server)
i.start(self.queue)
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, response_queue):
self.running = True
self.response_queue = response_queue
self.start_interfaces()
t = threading.Thread(target=self.process_requests_thread)
t.daemon = True
t.start()
self.blockchain.start()
threading.Thread.start(self)
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([ host, port, protocol ])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values(): i.stop()
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected:
self.switch_to_random_interface()
else:
if self.server_is_lagging():
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
while self.interfaces:
i = random.choice(self.interfaces.values())
if i.is_connected:
self.switch_to_interface(i)
break
else:
self.remove_interface(i)
def switch_to_interface(self, interface):
server = interface.server
print_error("switching to", server)
self.interface = interface
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.set_status('connected')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected:
self.stop_interface()
# notify gui
self.set_status('connecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface( self.interfaces[server] )
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def add_interface(self, i):
self.interfaces[i.server] = i
self.notify('interfaces')
def remove_interface(self, i):
self.interfaces.pop(i.server)
self.notify('interfaces')
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
if self.server_is_lagging():
print_error( "Server is lagging", blockchain_height, self.get_server_height())
if self.config.get('auto_cycle'):
self.set_server(i.server)
self.notify('updated')
def process_response(self, i, response):
method = response['method']
if method == 'blockchain.address.subscribe':
self.on_address(i, response)
elif method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.on_peers(i, response)
elif method == 'server.banner':
self.on_banner(i, response)
else:
self.response_queue.put(response)
def process_requests_thread(self):
while self.is_running():
try:
request = self.requests_queue.get(timeout=0.1)
except Queue.Empty:
continue
self.process_request(request)
def process_request(self, request):
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
except AttributeError:
out['error'] = "unknown method"
try:
out['result'] = f(*params)
except BaseException as e:
out['error'] = str(e)
print_error("network error", str(e))
self.response_queue.put(out)
return
if method == 'blockchain.address.subscribe':
addr = params[0]
if addr in self.addresses:
self.response_queue.put({'id':_id, 'result':self.addresses[addr]})
return
self.interface.send_request(request)
def run(self):
while self.is_running():
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
if len(self.interfaces) + len(self.pending_servers) < self.num_server:
self.start_random_interface()
if not self.interfaces:
if time.time() - self.disconnected_time > DISCONNECTED_RETRY_INTERVAL:
print_error('network: retrying connections')
self.disconnected_servers = set([])
self.disconnected_time = time.time()
if not self.interface.is_connected:
if time.time() - self.disconnected_time > DISCONNECTED_RETRY_INTERVAL:
print_error("forcing reconnection")
self.queue.put((self.interface, None))
self.disconnected_time = time.time()
continue
if response is not None:
self.process_response(i, response)
continue
# if response is None it is a notification about the interface
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected:
self.add_interface(i)
self.add_recent_server(i)
i.send_request({'method':'blockchain.headers.subscribe','params':[]})
if i == self.interface:
print_error('sending subscriptions to', self.interface.server)
self.send_subscriptions()
self.set_status('connected')
else:
self.disconnected_servers.add(i.server)
if i.server in self.interfaces:
self.remove_interface(i)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.set_status('disconnected')
if not self.interface.is_connected:
if self.config.get('auto_cycle'):
self.switch_to_random_interface()
else:
if self.default_server not in self.disconnected_servers:
print_error("restarting main interface")
if self.default_server in self.interfaces.keys():
self.switch_to_interface(self.interfaces[self.default_server])
else:
self.interface = self.start_interface(self.default_server)
print_error("Network: Stopping interfaces")
for i in self.interfaces.values():
i.stop()
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
if self.server_is_lagging() and self.config.get('auto_cycle'):
print_error( "Server lagging, stopping interface")
self.stop_interface()
self.notify('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.notify('servers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.notify('banner')
def on_address(self, i, r):
addr = r.get('params')[0]
result = r.get('result')
self.addresses[addr] = result
self.response_queue.put(r)
def stop(self):
print_error("stopping network")
with self.lock:
self.running = False
def is_running(self):
with self.lock:
return self.running
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote
electrum servers, each connection is handled by its own
thread object returned from Interface(). Its external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
for r in self.unanswered_requests.values():
self.interface.send_request(r)
for addr in self.subscribed_addresses:
self.interface.send_request({'method':'blockchain.address.subscribe','params':[addr]})
self.interface.send_request({'method':'server.banner','params':[]})
self.interface.send_request({'method':'server.peers.subscribe','params':[]})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect()
def auto_connect(self):
return self.config.get('auto_connect', False)
def get_interfaces(self):
'''The interfaces that are in connected state'''
return [s for s, i in self.interfaces.items() if i.is_connected()]
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if not server in self.interfaces.keys():
if server == self.default_server:
self.set_status('connecting')
i = interface.Interface(server, self.queue, self.config)
self.interfaces[i.server] = i
i.start()
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def start(self):
self.running = True
self.blockchain.start()
util.DaemonThread.start(self)
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for i in self.interfaces.values():
i.stop()
self.interface = None
self.interfaces = {}
def set_parameters(self, host, port, protocol, proxy, auto_connect):
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
servers = self.get_interfaces() # Those in connected state
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion = None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect():
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.print_error("starting %s; will switch once connected" % server)
self.start_interface(server)
return
i = self.interfaces[server]
if not i.is_connected():
# do nothing; we will switch once connected
return
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.stop_interface()
self.interface = i
self.addr_responses = {}
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
if self.interface:
self.interface.stop()
self.interface = None
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_if_notification(self, i):
'''Handle interface addition and removal through notifications'''
if i.is_connected():
self.add_recent_server(i)
i.send_request({'method':'blockchain.headers.subscribe','params':[]})
if i.server == self.default_server:
self.switch_to_interface(i.server)
else:
self.interfaces.pop(i.server, None)
self.heights.pop(i.server, None)
if i == self.interface:
self.interface = None
self.addr_responses = {}
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
# Our set of interfaces changed
self.notify('interfaces')
def process_response(self, i, response):
# the id comes from the daemon or the network proxy
_id = response.get('id')
if _id is not None:
if i != self.interface:
return
self.unanswered_requests.pop(_id)
method = response.get('method')
result = response.get('result')
if method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
self.banner = result
self.notify('banner')
elif method == 'blockchain.address.subscribe':
addr = response.get('params')[0]
self.addr_responses[addr] = result
self.response_queue.put(response)
else:
self.response_queue.put(response)
def handle_requests(self):
'''Some requests require connectivity, others we handle locally in
process_request() and must do so in order to e.g. prevent the
daemon seeming unresponsive.
'''
unhandled = []
while not self.requests_queue.empty():
request = self.requests_queue.get()
if not self.process_request(request):
unhandled.append(request)
for request in unhandled:
self.requests_queue.put(request)
def process_request(self, request):
'''Returns true if the request was processed.'''
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
out['result'] = f(*params)
except AttributeError:
out['error'] = "unknown method"
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return True
if method == 'blockchain.address.subscribe':
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({'id':_id, 'result':self.addr_responses[addr]})
return True
# This request needs connectivity. If we don't have an
# interface, we cannot process it.
if not self.is_connected():
return False
self.unanswered_requests[_id] = request
self.interface.send_request(request)
return True
def check_interfaces(self):
now = time.time()
# nodes
if len(self.interfaces) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def run(self):
while self.is_running():
self.check_interfaces()
self.handle_requests()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
# if response is None it is a notification about the interface
if response is None:
self.process_if_notification(i)
else:
self.process_response(i, response)
self.stop_network()
self.print_error("stopped")
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote electrum
servers, each connected socket is handled by an Interface() object.
Connections are initiated by a Connection() thread which stops once
the connection succeeds or fails.
Our external API:
- Member functions get_header(), get_interfaces(), get_local_height(),
get_parameters(), get_server_height(), get_status_value(),
is_connected(), new_blockchain_height(), set_parameters(),
stop()
"""
def __init__(self, config=None, plugins=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.lock = Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.subscriptions = defaultdict(list)
self.callbacks = defaultdict(list)
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.start_network(
deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
self.plugins = plugins
if self.plugins:
self.plugins.set_network(self)
def register_callback(self, event, callback):
with self.lock:
self.callbacks[event].append(callback)
def trigger_callback(self, event, params=()):
with self.lock:
callbacks = self.callbacks[event][:]
[callback(*params) for callback in callbacks]
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' %
(self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface is not None
def is_connecting(self):
return self.connection_status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def queue_request(self, method, params, interface=None):
# If you want to queue a request on any interface it must go
# through this function so message ids are properly tracked
if interface is None:
interface = self.interface
message_id = self.message_id
self.message_id += 1
if self.debug:
self.print_error(interface.host, "-->", method, params, message_id)
interface.queue_request(method, params, message_id)
return message_id
def send_subscriptions(self):
self.print_error('sending subscriptions to', self.interface.server,
len(self.unanswered_requests),
len(self.subscribed_addresses))
# Resend unanswered requests
requests = self.unanswered_requests.values()
self.unanswered_requests = {}
for request in requests:
message_id = self.queue_request(request[0], request[1])
self.unanswered_requests[message_id] = request
for addr in self.subscribed_addresses:
self.queue_request('blockchain.address.subscribe', [addr])
self.queue_request('server.banner', [])
self.queue_request('server.peers.subscribe', [])
self.queue_request('blockchain.estimatefee', [2])
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'fee':
value = self.fee
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
if key in ['status', 'updated']:
self.trigger_callback(key)
else:
self.trigger_callback(key, (value, ))
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect
def get_interfaces(self):
'''The interfaces that are in connected state'''
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if (not server in self.interfaces and not server in self.connecting):
if server == self.default_server:
self.print_error("connecting to %s as new interface" % server)
self.set_status('connecting')
self.connecting.add(server)
c = Connection(server, self.socket_queue, self.config.path)
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol,
exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"],
int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(
socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
assert not self.connecting and self.socket_queue.empty()
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for interface in self.interfaces.values():
self.close_interface(interface)
assert self.interface is None
assert not self.interfaces
self.connecting = set()
# Get a new queue - no old pending connections thanks!
self.socket_queue = Queue.Queue()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server = serialize_server(host, port, protocol)
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server, True)
# abort if changes were not allowed by config
if self.config.get('server') != server or self.config.get(
'proxy') != proxy_str:
return
self.auto_connect = auto_connect
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
'''Switch to a random connected server other than the current one'''
servers = self.get_interfaces() # Those in connected state
if self.default_server in servers:
servers.remove(self.default_server)
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion=None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
if suggestion and self.protocol == deserialize_server(
suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.interface = None
self.start_interface(server)
return
i = self.interfaces[server]
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.close_interface(self.interface)
self.interface = i
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def close_interface(self, interface):
if interface:
self.interfaces.pop(interface.server)
if interface.server == self.default_server:
self.interface = None
interface.close()
def add_recent_server(self, server):
# list is ordered
if server in self.recent_servers:
self.recent_servers.remove(server)
self.recent_servers.insert(0, server)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_response(self, interface, response, callback):
if self.debug:
self.print_error("<--", response)
error = response.get('error')
result = response.get('result')
method = response.get('method')
# We handle some responses; return the rest to the client.
if method == 'server.version':
interface.server_version = result
elif method == 'blockchain.headers.subscribe':
if error is None:
self.on_header(interface, result)
elif method == 'server.peers.subscribe':
if error is None:
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
if error is None:
self.banner = result
self.notify('banner')
elif method == 'blockchain.estimatefee':
if error is None:
self.fee = int(result * COIN)
self.print_error("recommended fee", self.fee)
self.notify('fee')
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(interface, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(interface, response)
else:
if callback is None:
params = response['params']
with self.lock:
for k, v in self.subscriptions.items():
if (method, params) in v:
callback = k
break
if callback is None:
self.print_error("received unexpected notification", method,
params)
else:
callback(response)
def process_responses(self, interface):
responses = interface.get_responses()
for request, response in responses:
callback = None
if request:
method, params, message_id = request
# client requests go through self.send() with a
# callback, are only sent to the current interface,
# and are placed in the unanswered_requests dictionary
client_req = self.unanswered_requests.pop(message_id, None)
if client_req:
assert interface == self.interface
callback = client_req[2]
# Copy the request method and params to the response
response['method'] = method
response['params'] = params
# Only once we've received a response to an addr subscription
# add it to the list; avoids double-sends on reconnection
if method == 'blockchain.address.subscribe':
self.subscribed_addresses.add(params[0])
else:
if not response: # Closed remotely / misbehaving
self.connection_down(interface.server)
break
# Rewrite response shape to match subscription request response
method = response.get('method')
params = response.get('params')
if method == 'blockchain.headers.subscribe':
response['result'] = params[0]
response['params'] = []
elif method == 'blockchain.address.subscribe':
response['params'] = [params[0]] # addr
response['result'] = params[1]
# Response is now in canonical form
self.process_response(interface, response, callback)
def send(self, messages, callback):
'''Messages is a list of (method, value) tuples'''
with self.lock:
self.pending_sends.append((messages, callback))
def process_pending_sends(self):
# Requests needs connectivity. If we don't have an interface,
# we cannot process them.
if not self.interface:
return
with self.lock:
sends = self.pending_sends
self.pending_sends = []
for messages, callback in sends:
subs = filter(lambda (m, v): m.endswith('.subscribe'), messages)
with self.lock:
for sub in subs:
if sub not in self.subscriptions[callback]:
self.subscriptions[callback].append(sub)
for method, params in messages:
message_id = self.queue_request(method, params)
self.unanswered_requests[message_id] = method, params, callback
def connection_down(self, server):
'''A connection to server either went down, or was never made.
We distinguish by whether it is in self.interfaces.'''
self.disconnected_servers.add(server)
if server == self.default_server:
self.set_status('disconnected')
if server in self.interfaces:
self.close_interface(self.interfaces[server])
self.heights.pop(server, None)
self.notify('interfaces')
def new_interface(self, server, socket):
self.add_recent_server(server)
self.interfaces[server] = interface = Interface(server, socket)
self.queue_request('blockchain.headers.subscribe', [], interface)
if server == self.default_server:
self.switch_to_interface(server)
self.notify('interfaces')
def maintain_sockets(self):
'''Socket maintenance.'''
# Responses to connection attempts?
while not self.socket_queue.empty():
server, socket = self.socket_queue.get()
self.connecting.remove(server)
if socket:
self.new_interface(server, socket)
else:
self.connection_down(server)
# Send pings and shut down stale interfaces
for interface in self.interfaces.values():
if interface.has_timed_out():
self.connection_down(interface.server)
elif interface.ping_required():
params = [ELECTRUM_VERSION, PROTOCOL_VERSION]
self.queue_request('server.version', params, interface)
now = time.time()
# nodes
if len(self.interfaces) + len(self.connecting) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
if not self.is_connecting():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
self.queue_request('blockchain.block.get_chunk', [idx], interface)
data['chunk_idx'] = idx
data['req_time'] = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx,
response['result'])
# If not finished, get the next chunk
if idx < 0 or self.get_local_height() >= data['if_height']:
self.bc_requests.popleft()
self.notify('updated')
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
self.queue_request('blockchain.block.get_header', [height], interface)
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(
data['chain'], response['result'])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify('updated')
else:
interface.print_error(
"header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them,
if necessary.
'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) / 2016)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object.
'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface in self.interfaces.values():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
self.connection_down(interface.server)
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def wait_on_sockets(self):
# Python docs say Windows doesn't like empty selects.
# Sleep to prevent busy looping
if not self.interfaces:
time.sleep(0.1)
return
rin = [i for i in self.interfaces.values()]
win = [i for i in self.interfaces.values() if i.unsent_requests]
rout, wout, xout = select.select(rin, win, [], 0.1)
assert not xout
for interface in wout:
interface.send_requests()
for interface in rout:
self.process_responses(interface)
def run(self):
self.blockchain.init()
while self.is_running():
self.maintain_sockets()
self.wait_on_sockets()
self.handle_bc_requests()
self.run_jobs() # Synchronizer and Verifier
self.process_pending_sends()
self.stop_network()
if self.plugins:
self.plugins.set_network(None)
self.print_error("stopped")
def on_header(self, i, header):
height = header.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get('merkle_root')
self.utxo_roots[i.server] = header.get('utxo_root')
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {'if_height': height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
def synchronous_get(self, request, timeout=100000000):
queue = Queue.Queue()
self.send([request], queue.put)
r = queue.get(True, timeout)
if r.get('error'):
raise BaseException(r.get('error'))
return r.get('result')
class NetworkProxy(threading.Thread):
# connects to daemon
# sends requests, runs callbacks
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default arguments!
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.daemon_port = config.get('daemon_port', DAEMON_PORT)
self.message_id = 0
self.unanswered_requests = {}
self.subscriptions = {}
self.debug = False
self.lock = threading.Lock()
self.pending_transactions_for_notifications = []
def start(self, start_daemon=False):
daemon_started = False
while True:
try:
self.socket.connect(('', self.daemon_port))
threading.Thread.start(self)
return True
except socket.error:
if not start_daemon:
return False
elif not daemon_started:
print_stderr("Starting daemon [%s]" %
self.config.get('server'))
daemon_started = True
pid = os.fork()
if (pid == 0): # The first child.
os.chdir("/")
os.setsid()
os.umask(0)
pid2 = os.fork()
if (pid2 == 0): # Second child
server = NetworkServer(self.config)
try:
server.main_loop()
except KeyboardInterrupt:
print "Ctrl C - Stopping server"
sys.exit(1)
sys.exit(0)
else:
time.sleep(0.1)
def parse_json(self, message):
s = message.find('\n')
if s == -1:
return None, message
j = json.loads(message[0:s])
return j, message[s + 1:]
def run(self):
# read responses and trigger callbacks
message = ''
while True:
try:
data = self.socket.recv(1024)
except:
data = ''
if not data:
break
message += data
while True:
response, message = self.parse_json(message)
if response is not None:
self.process(response)
else:
break
print "NetworkProxy: exiting"
def process(self, response):
# runs callbacks
if self.debug: print "<--", response
msg_id = response.get('id')
with self.lock:
method, params, callback = self.unanswered_requests.pop(msg_id)
result = response.get('result')
callback(None, {
'method': method,
'params': params,
'result': result,
'id': msg_id
})
def subscribe(self, messages, callback):
# detect if it is a subscription
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in messages:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
self.send(messages, callback)
def send(self, messages, callback):
"""return the ids of the requests that we sent"""
out = ''
ids = []
for m in messages:
method, params = m
request = json.dumps({
'id': self.message_id,
'method': method,
'params': params
})
self.unanswered_requests[
self.message_id] = method, params, callback
ids.append(self.message_id)
if self.debug: print "-->", request
self.message_id += 1
out += request + '\n'
while out:
sent = self.socket.send(out)
out = out[sent:]
return ids
def synchronous_get(self, requests, timeout=100000000):
queue = Queue.Queue()
ids = self.send(requests, lambda i, x: queue.put(x))
id2 = ids[:]
res = {}
while ids:
r = queue.get(True, timeout)
_id = r.get('id')
if _id in ids:
ids.remove(_id)
res[_id] = r.get('result')
out = []
for _id in id2:
out.append(res[_id])
return out
def get_servers(self):
return self.synchronous_get([('network.get_servers', [])])[0]
def get_header(self, height):
return self.synchronous_get([('network.get_header', [height])])[0]
def get_local_height(self):
return self.synchronous_get([('network.get_local_height', [])])[0]
def is_connected(self):
return self.synchronous_get([('network.is_connected', [])])[0]
def is_up_to_date(self):
return self.synchronous_get([('network.is_up_to_date', [])])[0]
def main_server(self):
return self.synchronous_get([('network.main_server', [])])[0]
def stop(self):
return self.synchronous_get([('daemon.shutdown', [])])[0]
def trigger_callback(self, cb):
pass
class NetworkProxy(util.DaemonThread):
"""Proxy for communicating with the daemon or Network.
If the daemon is running when this is initialized,
this will create a socket pipe.
Otherwise, this will create a new Network instance."""
def __init__(self, socket, config=None):
if config is None:
config = {} # Do not use mutables as default arguments!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.message_id = 0
self.unanswered_requests = {}
self.subscriptions = {}
self.debug = False
self.lock = threading.Lock()
self.pending_transactions_for_notifications = []
self.callbacks = {}
if socket:
self.pipe = util.SocketPipe(socket)
self.network = None
else:
self.pipe = util.QueuePipe()
self.network = Network(self.pipe, config)
self.network.start()
for key in ['status','banner','updated','servers','interfaces']:
value = self.network.get_status_value(key)
self.pipe.get_queue.put({'method':'network.status', 'params':[key, value]})
# status variables
self.status = 'connecting'
self.servers = {}
self.banner = ''
self.blockchain_height = 0
self.server_height = 0
self.interfaces = []
def switch_to_active_chain(self):
"""Create a new Network instance or send message to daemon."""
with self.lock:
# for the network.switch_chains request
message_id = self.message_id
self.message_id = 0
self.unanswered_requests = {}
self.subscriptions = {}
self.pending_transactions_for_notifications = []
self.callbacks = {}
self.status = 'connecting'
self.servers = {}
self.banner = ''
self.blockchain_height = 0
self.server_height = 0
self.interfaces = []
# Not daemon, probably running GUI
if self.network:
self.network.switch_chains()
for key in ['status','banner','updated','servers','interfaces']:
value = self.network.get_status_value(key)
self.pipe.get_queue.put({'method':'network.status', 'params':[key, value]})
# Daemon is running
else:
req = {'id': message_id, 'method': 'network.switch_chains', 'params':[chainparams.get_active_chain().code]}
self.pipe.send(req)
def run(self):
while self.is_running():
try:
response = self.pipe.get()
except util.timeout:
continue
if response is None:
break
self.process(response)
self.trigger_callback('stop')
if self.network:
self.network.stop()
self.print_error("stopped")
def process(self, response):
if self.debug:
print_error("<--", response)
if response.get('method') == 'network.status':
key, value = response.get('params')
if key == 'status':
self.status = value
elif key == 'banner':
self.banner = value
elif key == 'updated':
self.blockchain_height, self.server_height = value
elif key == 'servers':
self.servers = value
elif key == 'interfaces':
self.interfaces = value
self.trigger_callback(key)
return
msg_id = response.get('id')
result = response.get('result')
error = response.get('error')
if msg_id is not None:
with self.lock:
try:
method, params, callback = self.unanswered_requests.pop(msg_id)
except KeyError:
return
else:
method = response.get('method')
params = response.get('params')
with self.lock:
for k,v in self.subscriptions.items():
if (method, params) in v:
callback = k
break
else:
print_error( "received unexpected notification", method, params)
return
r = {'method':method, 'params':params, 'result':result, 'id':msg_id, 'error':error}
callback(r)
def send(self, messages, callback):
"""return the ids of the requests that we sent"""
# detect subscriptions
sub = []
for message in messages:
m, v = message
if m[-10:] == '.subscribe':
sub.append(message)
if sub:
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in sub:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
with self.lock:
requests = []
ids = []
for m in messages:
method, params = m
request = { 'id':self.message_id, 'method':method, 'params':params }
self.unanswered_requests[self.message_id] = method, params, callback
ids.append(self.message_id)
requests.append(request)
if self.debug:
print_error("-->", request)
self.message_id += 1
self.pipe.send_all(requests)
return ids
def synchronous_get(self, requests, timeout=100000000):
queue = Queue.Queue()
ids = self.send(requests, queue.put)
id2 = ids[:]
res = {}
while ids:
r = queue.get(True, timeout)
_id = r.get('id')
ids.remove(_id)
if r.get('error'):
return BaseException(r.get('error'))
result = r.get('result')
res[_id] = r.get('result')
out = []
for _id in id2:
out.append(res[_id])
return out
def get_servers(self):
return self.servers
def get_interfaces(self):
return self.interfaces
def get_header(self, height):
return self.synchronous_get([('network.get_header', [height])])[0]
def get_local_height(self):
return self.blockchain_height
def get_server_height(self):
return self.server_height
def is_connected(self):
return self.status == 'connected'
def is_connecting(self):
return self.status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def get_parameters(self):
return self.synchronous_get([('network.get_parameters', [])])[0]
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server_str = serialize_server(host, port, protocol)
self.config.set_key('auto_connect', auto_connect, True)
self.config.set_key("proxy", proxy_str, True)
self.config.set_key("use_ssl", protocol == 's', True)
self.config.set_key("server", server_str, True)
# abort if changes were not allowed by config
if self.config.get('server') != server_str or self.config.get('proxy') != proxy_str:
return
return self.synchronous_get([('network.set_parameters', (host, port, protocol, proxy, auto_connect))])[0]
def stop_daemon(self):
return self.send([('daemon.stop',[])], None)
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event):
with self.lock:
callbacks = self.callbacks.get(event,[])[:]
if callbacks:
[callback() for callback in callbacks]
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote
electrum servers, each connection is handled by its own
thread object returned from Interface(). Its external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server('s')
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.pending_servers = set()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network
self.interface = None
self.interfaces = {}
self.start_network(
deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
h = self.get_server_height()
if not h:
self.print_error('no height for main interface')
return False
lag = self.get_local_height() - self.get_server_height()
return lag > 1
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error('sending subscriptions to', self.interface.server,
len(self.unanswered_requests),
len(self.subscribed_addresses))
for r in self.unanswered_requests.values():
self.interface.send_request(r)
for addr in self.subscribed_addresses:
self.interface.send_request({
'method': 'blockchain.address.subscribe',
'params': [addr]
})
self.interface.send_request({'method': 'server.banner', 'params': []})
self.interface.send_request({
'method': 'server.peers.subscribe',
'params': []
})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({
'method': 'network.status',
'params': [key, value]
})
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys(
):
continue
else:
choice_list.append(s)
if not choice_list:
return
server = random.choice(choice_list)
return server
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
auto_connect = self.config.get('auto_cycle', True)
return host, port, protocol, self.proxy, auto_connect
def get_interfaces(self):
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if not server in self.interfaces.keys():
if server == self.default_server:
self.set_status('connecting')
i = interface.Interface(server, self.queue, self.config)
self.pending_servers.add(server)
i.start()
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def start(self):
self.running = True
self.blockchain.start()
util.DaemonThread.start(self)
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"],
int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(
socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
# FIXME: this forgets to handle pending servers...
self.print_error("stopping network")
for i in self.interfaces.values():
i.stop()
self.interface = None
self.interfaces = {}
def set_parameters(self, host, port, protocol, proxy, auto_connect):
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
elif auto_connect and (not self.is_connected()
or self.server_is_lagging()):
self.switch_to_random_interface()
def switch_to_random_interface(self):
if self.interfaces:
server = random.choice(self.interfaces.keys())
self.switch_to_interface(server)
def switch_to_interface(self, server):
'''Switch to server as our interface. If not already connected, start a
connection - we will switch on receipt of the connection notification'''
self.default_server = server
if server in self.interfaces:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.stop_interface()
self.interface = self.interfaces[server]
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
elif server not in self.pending_servers:
self.print_error("starting %s; will switch once connected" %
server)
self.start_interface(server)
def stop_interface(self):
if self.interface:
self.interface.stop()
self.interface = None
def set_server(self, server):
if self.default_server == server and self.is_connected():
return
if self.protocol != deserialize_server(server)[2]:
return
self.switch_to_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0, s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
if self.server_is_lagging():
self.print_error("Server is lagging", blockchain_height,
self.get_server_height())
if self.config.get('auto_cycle'):
self.set_server(i.server)
self.notify('updated')
def process_if_notification(self, i):
'''Handle interface addition and removal through notifications'''
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected():
self.interfaces[i.server] = i
self.add_recent_server(i)
i.send_request({
'method': 'blockchain.headers.subscribe',
'params': []
})
if i.server == self.default_server:
self.switch_to_interface(i.server)
else:
self.interfaces.pop(i.server, None)
self.heights.pop(i.server, None)
if i == self.interface:
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
# Our set of interfaces changed
self.notify('interfaces')
def process_response(self, i, response):
# the id comes from the daemon or the network proxy
_id = response.get('id')
if _id is not None:
if i != self.interface:
return
self.unanswered_requests.pop(_id)
method = response.get('method')
result = response.get('result')
if method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
self.banner = result
self.notify('banner')
elif method == 'blockchain.address.subscribe':
addr = response.get('params')[0]
self.addr_responses[addr] = result
self.response_queue.put(response)
else:
self.response_queue.put(response)
def handle_requests(self):
while True:
try:
request = self.requests_queue.get_nowait()
except Queue.Empty:
break
self.process_request(request)
def process_request(self, request):
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id': _id}
try:
f = getattr(self, method[8:])
out['result'] = f(*params)
except AttributeError:
out['error'] = "unknown method"
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return
if method == 'blockchain.address.subscribe':
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({
'id': _id,
'result': self.addr_responses[addr]
})
return
# store unanswered request
self.unanswered_requests[_id] = request
self.interface.send_request(request)
def check_interfaces(self):
now = time.time()
# nodes
if len(self.interfaces) + len(self.pending_servers) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.config.get('auto_cycle'):
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def run(self):
while self.is_running():
self.check_interfaces()
self.handle_requests()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
# if response is None it is a notification about the interface
if response is None:
self.process_if_notification(i)
else:
self.process_response(i, response)
self.stop_network()
self.print_error("stopped")
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i, result))
if i == self.interface:
if self.server_is_lagging() and self.config.get('auto_cycle'):
self.print_error("Server lagging, stopping interface")
self.stop_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(threading.Thread):
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if isinstance(config,
dict) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.protocol = self.config.get('protocol', 's')
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.disconnected_time = time.time()
self.recent_servers = self.config.get('recent_servers',
[]) # successful connections
self.pending_servers = set()
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# address subscriptions and cached results
self.addresses = {}
self.connection_status = 'connecting'
self.requests_queue = Queue.Queue()
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
h = self.get_server_height()
if not h:
print_error('no height for main interface')
return False
lag = self.get_local_height() - self.get_server_height()
return lag > 1
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected
def send_subscriptions(self):
for addr in self.addresses:
self.interface.send_request({
'method': 'blockchain.address.subscribe',
'params': [addr]
})
self.interface.send_request({'method': 'server.banner', 'params': []})
self.interface.send_request({
'method': 'server.peers.subscribe',
'params': []
})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({
'method': 'network.status',
'params': [key, value]
})
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys(
):
continue
else:
choice_list.append(s)
if not choice_list:
return
server = random.choice(choice_list)
return server
def get_parameters(self):
host, port, protocol = self.default_server.split(':')
proxy = self.proxy
auto_connect = self.config.get('auto_cycle', True)
return host, port, protocol, proxy, auto_connect
def get_interfaces(self):
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.pending_servers.add(server)
i.start(self.queue)
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, response_queue):
self.running = True
self.response_queue = response_queue
self.start_interfaces()
t = threading.Thread(target=self.process_requests_thread)
t.daemon = True
t.start()
self.blockchain.start()
threading.Thread.start(self)
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([host, port, protocol])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values():
i.stop()
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected:
self.switch_to_random_interface()
else:
if self.server_is_lagging():
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
while self.interfaces:
i = random.choice(self.interfaces.values())
if i.is_connected:
self.switch_to_interface(i)
break
else:
self.remove_interface(i)
def switch_to_interface(self, interface):
server = interface.server
print_error("switching to", server)
self.interface = interface
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.set_status('connected')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected:
self.stop_interface()
# notify gui
self.set_status('connecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface(self.interfaces[server])
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0, s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def add_interface(self, i):
self.interfaces[i.server] = i
self.notify('interfaces')
def remove_interface(self, i):
self.interfaces.pop(i.server)
self.notify('interfaces')
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
if self.server_is_lagging():
print_error("Server is lagging", blockchain_height,
self.get_server_height())
if self.config.get('auto_cycle'):
self.set_server(i.server)
self.notify('updated')
def process_response(self, i, response):
method = response['method']
if method == 'blockchain.address.subscribe':
self.on_address(i, response)
elif method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.on_peers(i, response)
elif method == 'server.banner':
self.on_banner(i, response)
else:
self.response_queue.put(response)
def process_requests_thread(self):
while self.is_running():
try:
request = self.requests_queue.get(timeout=0.1)
except Queue.Empty:
continue
self.process_request(request)
def process_request(self, request):
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id': _id}
try:
f = getattr(self, method[8:])
except AttributeError:
out['error'] = "unknown method"
try:
out['result'] = f(*params)
except BaseException as e:
out['error'] = str(e)
print_error("network error", str(e))
self.response_queue.put(out)
return
if method == 'blockchain.address.subscribe':
addr = params[0]
if addr in self.addresses:
self.response_queue.put({
'id': _id,
'result': self.addresses[addr]
})
return
self.interface.send_request(request)
def run(self):
while self.is_running():
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
if len(self.interfaces) + len(
self.pending_servers) < self.num_server:
self.start_random_interface()
if not self.interfaces:
if time.time(
) - self.disconnected_time > DISCONNECTED_RETRY_INTERVAL:
print_error('network: retrying connections')
self.disconnected_servers = set([])
self.disconnected_time = time.time()
continue
if response is not None:
self.process_response(i, response)
continue
# if response is None it is a notification about the interface
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected:
self.add_interface(i)
self.add_recent_server(i)
i.send_request({
'method': 'blockchain.headers.subscribe',
'params': []
})
if i == self.interface:
print_error('sending subscriptions to',
self.interface.server)
self.send_subscriptions()
self.set_status('connected')
else:
self.disconnected_servers.add(i.server)
if i.server in self.interfaces:
self.remove_interface(i)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.set_status('disconnected')
if not self.interface.is_connected:
if self.config.get('auto_cycle'):
self.switch_to_random_interface()
else:
if self.default_server not in self.disconnected_servers:
print_error("restarting main interface")
if self.default_server in self.interfaces.keys():
self.switch_to_interface(
self.interfaces[self.default_server])
else:
self.interface = self.start_interface(
self.default_server)
print_error("Network: Stopping interfaces")
for i in self.interfaces.values():
i.stop()
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i, result))
if i == self.interface:
if self.server_is_lagging() and self.config.get('auto_cycle'):
print_error("Server lagging, stopping interface")
self.stop_interface()
self.notify('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.notify('servers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.notify('banner')
def on_address(self, i, r):
addr = r.get('params')[0]
result = r.get('result')
self.addresses[addr] = result
self.response_queue.put(r)
def stop(self):
print_error("stopping network")
with self.lock:
self.running = False
def is_running(self):
with self.lock:
return self.running
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote
electrum servers, each connection is handled by its own
thread object returned from Interface(). Its external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.start_network(
deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' %
(self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error('sending subscriptions to', self.interface.server,
len(self.unanswered_requests),
len(self.subscribed_addresses))
for r in self.unanswered_requests.values():
self.interface.send_request(r)
for addr in self.subscribed_addresses:
self.interface.send_request({
'method': 'blockchain.address.subscribe',
'params': [addr]
})
self.interface.send_request({'method': 'server.banner', 'params': []})
self.interface.send_request({
'method': 'server.peers.subscribe',
'params': []
})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({
'method': 'network.status',
'params': [key, value]
})
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect
def get_interfaces(self):
'''The interfaces that are in connected state'''
return [s for s, i in self.interfaces.items() if i.is_connected()]
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if not server in self.interfaces.keys():
if server == self.default_server:
self.set_status('connecting')
i = interface.Interface(server, self.queue, self.config)
self.interfaces[i.server] = i
i.start()
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol,
exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"],
int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(
socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for i in self.interfaces.values():
i.stop()
self.interface = None
self.interfaces = {}
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.auto_connect = auto_connect
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
servers = self.get_interfaces() # Those in connected state
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion=None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
if suggestion and self.protocol == deserialize_server(
suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.print_error("starting %s; will switch once connected" %
server)
self.start_interface(server)
return
i = self.interfaces[server]
if not i.is_connected():
# do nothing; we will switch once connected
return
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.stop_interface()
self.interface = i
self.addr_responses = {}
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
if self.interface:
self.interface.stop()
self.interface = None
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0, s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_if_notification(self, i):
'''Handle interface addition and removal through notifications'''
if i.is_connected():
self.add_recent_server(i)
i.send_request({
'method': 'blockchain.headers.subscribe',
'params': []
})
if i.server == self.default_server:
self.switch_to_interface(i.server)
else:
self.interfaces.pop(i.server, None)
self.heights.pop(i.server, None)
if i == self.interface:
self.interface = None
self.addr_responses = {}
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
# Our set of interfaces changed
self.notify('interfaces')
def process_response(self, i, response):
# the id comes from the daemon or the network proxy
_id = response.get('id')
if _id is not None:
if i != self.interface:
return
self.unanswered_requests.pop(_id)
method = response.get('method')
result = response.get('result')
if method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
self.banner = result
self.notify('banner')
elif method == 'blockchain.address.subscribe':
addr = response.get('params')[0]
self.addr_responses[addr] = result
self.response_queue.put(response)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(i, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(i, response)
else:
self.response_queue.put(response)
def handle_requests(self):
'''Some requests require connectivity, others we handle locally in
process_request() and must do so in order to e.g. prevent the
daemon seeming unresponsive.
'''
unhandled = []
while not self.requests_queue.empty():
request = self.requests_queue.get()
if not self.process_request(request):
unhandled.append(request)
for request in unhandled:
self.requests_queue.put(request)
def process_request(self, request):
'''Returns true if the request was processed.'''
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id': _id}
try:
f = getattr(self, method[8:])
out['result'] = f(*params)
except AttributeError:
out['error'] = "unknown method"
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return True
if method == 'blockchain.address.subscribe':
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({
'id': _id,
'result': self.addr_responses[addr]
})
return True
# This request needs connectivity. If we don't have an
# interface, we cannot process it.
if not self.is_connected():
return False
self.unanswered_requests[_id] = request
self.interface.send_request(request)
return True
def check_interfaces(self):
now = time.time()
# nodes
if len(self.interfaces) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
interface.send_request({
'method': 'blockchain.block.get_chunk',
'params': [idx]
})
data['chunk_idx'] = idx
data['req_time'] = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx,
response['result'])
# If not finished, get the next chunk
if idx < 0 or self.get_local_height() >= data['if_height']:
self.bc_requests.popleft()
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
interface.send_request({
'method': 'blockchain.block.get_header',
'params': [height]
})
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(
data['chain'], response['result'])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify('updated')
else:
interface.print_error(
"header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them, if necessary'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) / 2016)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface.is_connected():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
interface.stop()
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def run(self):
self.blockchain.init()
while self.is_running():
self.check_interfaces()
self.handle_requests()
self.handle_bc_requests()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
# if response is None it is a notification about the interface
if response is None:
self.process_if_notification(i)
else:
self.process_response(i, response)
self.stop_network()
self.print_error("stopped")
def on_header(self, i, r):
header = r.get('result')
if not header:
return
height = header.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get('merkle_root')
self.utxo_roots[i.server] = header.get('utxo_root')
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {'if_height': height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(util.DaemonThread):
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server('s')
self.protocol = deserialize_server(self.default_server)[2]
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.recent_servers = self.read_recent_servers()
self.pending_servers = set()
self.banner = ''
self.interface = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# address subscriptions
self.addresses = set()
# cached results
self.addr_responses = {}
self.connection_status = 'connecting'
self.requests_queue = Queue.Queue()
self.set_proxy(deserialize_proxy(self.config.get('proxy')))
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
h = self.get_server_height()
if not h:
self.print_error('no height for main interface')
return False
lag = self.get_local_height() - self.get_server_height()
return lag > 1
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
self.print_error('sending subscriptions to', self.interface.server, len(self.addresses))
for addr in self.addresses:
self.interface.send_request({'method':'blockchain.address.subscribe', 'params':[addr]})
self.interface.send_request({'method':'server.banner','params':[]})
self.interface.send_request({'method':'server.peers.subscribe','params':[]})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list:
return
server = random.choice( choice_list )
return server
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
auto_connect = self.config.get('auto_cycle', True)
return host, port, protocol, self.proxy, auto_connect
def get_interfaces(self):
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.queue, self.config)
self.pending_servers.add(server)
i.start()
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, response_queue):
self.running = True
self.response_queue = response_queue
self.start_interfaces()
t = threading.Thread(target=self.process_requests_thread)
t.start()
self.blockchain.start()
util.DaemonThread.start(self)
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def set_parameters(self, host, port, protocol, proxy, auto_connect):
if self.proxy != proxy or self.protocol != protocol:
self.print_error('restarting network')
for i in self.interfaces.values():
i.stop()
self.interfaces.pop(i.server)
self.set_proxy(proxy)
self.protocol = protocol
self.disconnected_servers = set([])
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected():
self.switch_to_random_interface()
else:
if self.server_is_lagging():
self.stop_interface()
else:
server_str = serialize_server(host, port, protocol)
self.set_server(server_str)
def switch_to_random_interface(self):
while self.interfaces:
i = random.choice(self.interfaces.values())
if i.is_connected():
self.switch_to_interface(i)
break
else:
self.remove_interface(i)
def switch_to_interface(self, interface):
server = interface.server
self.print_error("switching to", server)
self.interface = interface
self.default_server = server
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected():
return
if self.protocol != deserialize_server(server)[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected():
self.stop_interface()
# notify gui
self.set_status('connecting')
# start interface
self.default_server = server
if server in self.interfaces.keys():
self.switch_to_interface( self.interfaces[server] )
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def add_interface(self, i):
self.interfaces[i.server] = i
self.notify('interfaces')
def remove_interface(self, i):
self.interfaces.pop(i.server)
self.notify('interfaces')
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
if self.server_is_lagging():
self.print_error("Server is lagging", blockchain_height, self.get_server_height())
if self.config.get('auto_cycle'):
self.set_server(i.server)
self.notify('updated')
def process_response(self, i, response):
method = response['method']
if method == 'blockchain.address.subscribe':
self.on_address(i, response)
elif method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.on_peers(i, response)
elif method == 'server.banner':
self.on_banner(i, response)
else:
self.response_queue.put(response)
def process_requests_thread(self):
while self.is_running():
try:
request = self.requests_queue.get(timeout=0.1)
except Queue.Empty:
continue
self.process_request(request)
def process_request(self, request):
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
except AttributeError:
out['error'] = "unknown method"
try:
out['result'] = f(*params)
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return
if method == 'blockchain.address.subscribe':
addr = params[0]
self.addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({'id':_id, 'result':self.addr_responses[addr]})
return
try:
self.interface.send_request(request)
except:
# put it back in the queue
self.print_error("warning: interface not ready for", request)
self.requests_queue.put(request)
time.sleep(0.1)
def check_interfaces(self):
now = time.time()
if len(self.interfaces) + len(self.pending_servers) < self.num_server:
self.start_random_interface()
if not self.interfaces:
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
if not self.interface.is_connected():
if self.config.get('auto_cycle'):
if self.interfaces:
self.switch_to_random_interface()
else:
if self.default_server in self.interfaces.keys():
self.switch_to_interface(self.interfaces[self.default_server])
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
if self.default_server not in self.pending_servers:
self.print_error("forcing reconnection")
self.interface = self.start_interface(self.default_server)
def run(self):
while self.is_running():
self.check_interfaces()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
if response is not None:
self.process_response(i, response)
continue
# if response is None it is a notification about the interface
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected():
self.add_interface(i)
self.add_recent_server(i)
i.send_request({'method':'blockchain.headers.subscribe','params':[]})
if i == self.interface:
self.send_subscriptions()
self.set_status('connected')
else:
if i.server in self.interfaces:
self.remove_interface(i)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
self.print_error("stopping interfaces")
for i in self.interfaces.values():
i.stop()
self.print_error("stopped")
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
if self.server_is_lagging() and self.config.get('auto_cycle'):
self.print_error("Server lagging, stopping interface")
self.stop_interface()
self.notify('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.notify('servers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.notify('banner')
def on_address(self, i, r):
addr = r.get('params')[0]
result = r.get('result')
self.addr_responses[addr] = result
self.response_queue.put(r)
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(threading.Thread):
def __init__(self, config={}):
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.lock = threading.Lock()
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.callbacks = {}
self.protocol = self.config.get('protocol', 's')
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = [] # returned by interface (list from irc)
self.disconnected_servers = []
self.recent_servers = self.config.get('recent_servers',
[]) # successful connections
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.server_lag = 0
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# default subscriptions
self.subscriptions = {}
self.subscriptions[self.on_banner] = [('server.banner', [])]
self.subscriptions[self.on_peers] = [('server.peers.subscribe', [])]
def is_connected(self):
return self.interface and self.interface.is_connected
def send_subscriptions(self):
for cb, sub in self.subscriptions.items():
self.interface.send(sub, cb)
def subscribe(self, messages, callback):
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in messages:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
if self.interface and self.interface.is_connected:
self.interface.send(messages, callback)
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event):
with self.lock:
callbacks = self.callbacks.get(event, [])[:]
if callbacks:
[callback() for callback in callbacks]
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list:
if not self.interfaces:
# we are probably offline, retry later
self.disconnected_servers = []
return
server = random.choice(choice_list)
return server
def get_servers(self):
out = self.irc_servers if self.irc_servers else DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.interfaces[server] = i
i.start(self.queue)
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
self.interface = self.interfaces[self.default_server]
for i in range(NUM_SERVERS):
self.start_random_interface()
if not self.interface:
self.interface = self.interfaces.values()[0]
def start(self, wait=False):
self.start_interfaces()
threading.Thread.start(self)
if wait:
self.interface.connect_event.wait()
return self.interface.is_connected
def wait_until_connected(self):
while not self.interface:
time.sleep(1)
self.interface.connect_event.wait()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([host, port, protocol])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values():
i.stop()
if auto_connect:
self.interface = None
return
if auto_connect:
if not self.interface:
self.switch_to_random_interface()
else:
if self.server_lag > 0:
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
if self.interfaces:
self.switch_to_interface(random.choice(self.interfaces.values()))
def switch_to_interface(self, interface):
assert self.interface is None
server = interface.server
print_error("switching to", server)
self.interface = interface
h = self.heights.get(server)
if h:
self.server_lag = self.blockchain.height() - h
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.trigger_callback('connected')
def stop_interface(self):
self.interface.stop()
self.interface = None
def set_server(self, server):
if self.default_server == server and self.interface:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface:
self.stop_interface()
# notify gui
self.trigger_callback('disconnecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface(self.interfaces[server])
else:
self.start_interface(server)
self.interface = self.interfaces[server]
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0, s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
h = self.heights.get(self.interface.server)
if h:
self.server_lag = blockchain_height - h
if self.server_lag > 1:
print_error("Server is lagging", blockchain_height, h)
if self.config.get('auto_cycle'):
self.set_server(i.server)
else:
print_error('no height for main interface')
self.trigger_callback('updated')
def run(self):
self.blockchain.start()
with self.lock:
self.running = True
while self.is_running():
try:
i = self.queue.get(timeout=30 if self.interfaces else 3)
except Queue.Empty:
if len(self.interfaces) < NUM_SERVERS:
self.start_random_interface()
continue
if i.is_connected:
self.add_recent_server(i)
i.send([('blockchain.headers.subscribe', [])], self.on_header)
if i == self.interface:
print_error('sending subscriptions to',
self.interface.server)
self.send_subscriptions()
self.trigger_callback('connected')
else:
self.disconnected_servers.append(i.server)
self.interfaces.pop(i.server)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.interface = None
self.trigger_callback('disconnected')
if self.interface is None and self.config.get('auto_cycle'):
self.switch_to_random_interface()
def on_header(self, i, r):
result = r.get('result')
if not result: return
height = result.get('block_height')
self.heights[i.server] = height
# notify blockchain about the new height
self.blockchain.queue.put((i, result))
if i == self.interface:
self.server_lag = self.blockchain.height() - height
if self.server_lag > 1 and self.config.get('auto_cycle'):
print_error("Server lagging, stopping interface")
self.stop_interface()
self.trigger_callback('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = self.parse_servers(r.get('result'))
self.trigger_callback('peers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.trigger_callback('banner')
def stop(self):
with self.lock:
self.running = False
def is_running(self):
with self.lock:
return self.running
def synchronous_get(self, requests, timeout=100000000):
queue = Queue.Queue()
ids = self.interface.send(requests, lambda i, r: queue.put(r))
id2 = ids[:]
res = {}
while ids:
r = queue.get(True, timeout)
_id = r.get('id')
if _id in ids:
ids.remove(_id)
res[_id] = r.get('result')
out = []
for _id in id2:
out.append(res[_id])
return out
def retrieve_transaction(self, tx_hash, tx_height=0):
import transaction
r = self.synchronous_get([('blockchain.transaction.get',
[tx_hash, tx_height])])[0]
if r:
return transaction.Transaction(r)
def parse_servers(self, result):
""" parse servers list into dict format"""
from version import PROTOCOL_VERSION
servers = {}
for item in result:
host = item[1]
out = {}
version = None
pruning_level = '-'
if len(item) > 2:
for v in item[2]:
if re.match("[stgh]\d*", v):
protocol, port = v[0], v[1:]
if port == '': port = DEFAULT_PORTS[protocol]
out[protocol] = port
elif re.match("v(.?)+", v):
version = v[1:]
elif re.match("p\d*", v):
pruning_level = v[1:]
if pruning_level == '': pruning_level = '0'
try:
is_recent = float(version) >= float(PROTOCOL_VERSION)
except:
is_recent = False
if out and is_recent:
out['pruning'] = pruning_level
servers[host] = out
return servers
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote electrum
servers, each connected socket is handled by an Interface() object.
Connections are initiated by a Connection() thread which stops once
the connection succeeds or fails.
Our external API:
- Member functions get_header(), get_interfaces(), get_local_height(),
get_parameters(), get_server_height(), get_status_value(),
is_connected(), set_parameters(), stop()
"""
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 10 if not self.config.get('oneserver') else 0
self.blockchains = read_blockchains(self.config)
self.print_error("blockchains", self.blockchains.keys())
self.blockchain_index = config.get('blockchain_index', 0)
if self.blockchain_index not in self.blockchains.keys():
self.blockchain_index = 0
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.lock = threading.Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.donation_address = ''
self.relay_fee = None
self.headers = {}
# callbacks passed with subscriptions
self.subscriptions = defaultdict(list)
self.sub_cache = {}
# callbacks set by the GUI
self.callbacks = defaultdict(list)
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', True)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def register_callback(self, callback, events):
with self.lock:
for event in events:
self.callbacks[event].append(callback)
def unregister_callback(self, callback):
with self.lock:
for callbacks in self.callbacks.values():
if callback in callbacks:
callbacks.remove(callback)
def trigger_callback(self, event, *args):
with self.lock:
callbacks = self.callbacks[event][:]
[callback(event, *args) for callback in callbacks]
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
h = self.headers.get(self.default_server)
return h['block_height'] if h else 0
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return True
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface is not None
def is_connecting(self):
return self.connection_status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def queue_request(self, method, params, interface=None):
# If you want to queue a request on any interface it must go
# through this function so message ids are properly tracked
if interface is None:
interface = self.interface
message_id = self.message_id
self.message_id += 1
if self.debug:
self.print_error(interface.host, "-->", method, params, message_id)
interface.queue_request(method, params, message_id)
return message_id
def send_subscriptions(self):
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
self.sub_cache.clear()
# Resend unanswered requests
requests = self.unanswered_requests.values()
self.unanswered_requests = {}
for request in requests:
message_id = self.queue_request(request[0], request[1])
self.unanswered_requests[message_id] = request
self.queue_request('server.banner', [])
self.queue_request('server.donation_address', [])
self.queue_request('server.peers.subscribe', [])
for i in bitcoin.FEE_TARGETS:
self.queue_request('blockchain.estimatefee', [i])
self.queue_request('blockchain.relayfee', [])
for addr in self.subscribed_addresses:
self.queue_request('blockchain.address.subscribe', [addr])
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'fee':
value = self.config.fee_estimates
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
if key in ['status', 'updated']:
self.trigger_callback(key)
else:
self.trigger_callback(key, self.get_status_value(key))
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect
def get_donation_address(self):
if self.is_connected():
return self.donation_address
def get_interfaces(self):
'''The interfaces that are in connected state'''
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers.copy()
out.update(DEFAULT_SERVERS)
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if (not server in self.interfaces and not server in self.connecting):
if server == self.default_server:
self.print_error("connecting to %s as new interface" % server)
self.set_status('connecting')
self.connecting.add(server)
c = Connection(server, self.socket_queue, self.config.path)
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
self.print_error('setting proxy', proxy)
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode,
proxy["host"],
int(proxy["port"]),
# socks.py seems to want either None or a non-empty string
username=(proxy.get("user", "") or None),
password=(proxy.get("password", "") or None))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
assert not self.connecting and self.socket_queue.empty()
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for interface in self.interfaces.values():
self.close_interface(interface)
if self.interface:
self.close_interface(self.interface)
assert self.interface is None
assert not self.interfaces
self.connecting = set()
# Get a new queue - no old pending connections thanks!
self.socket_queue = Queue.Queue()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server = serialize_server(host, port, protocol)
# sanitize parameters
try:
deserialize_server(serialize_server(host, port, protocol))
if proxy:
proxy_modes.index(proxy["mode"]) + 1
int(proxy['port'])
except:
return
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server, True)
# abort if changes were not allowed by config
if self.config.get('server') != server or self.config.get('proxy') != proxy_str:
return
self.auto_connect = auto_connect
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
'''Switch to a random connected server other than the current one'''
servers = self.get_interfaces() # Those in connected state
if self.default_server in servers:
servers.remove(self.default_server)
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
# switch to one that has the correct header (not height)
header = self.blockchain().read_header(self.get_local_height())
filtered = map(lambda x:x[0], filter(lambda x: x[1]==header, self.headers.items()))
if filtered:
choice = random.choice(filtered)
self.switch_to_interface(choice)
self.notify('updated')
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.interface = None
self.start_interface(server)
return
i = self.interfaces[server]
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
# fixme: we don't want to close headers sub
#self.close_interface(self.interface)
self.interface = i
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def close_interface(self, interface):
if interface:
if interface.server in self.interfaces:
self.interfaces.pop(interface.server)
if interface.server == self.default_server:
self.interface = None
interface.close()
def add_recent_server(self, server):
# list is ordered
if server in self.recent_servers:
self.recent_servers.remove(server)
self.recent_servers.insert(0, server)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def process_response(self, interface, response, callbacks):
if self.debug:
self.print_error("<--", response)
error = response.get('error')
result = response.get('result')
method = response.get('method')
params = response.get('params')
# We handle some responses; return the rest to the client.
if method == 'server.version':
interface.server_version = result
elif method == 'blockchain.headers.subscribe':
if error is None:
self.on_notify_header(interface, result)
elif method == 'server.peers.subscribe':
if error is None:
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
if error is None:
self.banner = result
self.notify('banner')
elif method == 'server.donation_address':
if error is None:
self.donation_address = result
elif method == 'blockchain.estimatefee':
if error is None and result > 0:
i = params[0]
fee = int(result*COIN)
self.config.fee_estimates[i] = fee
self.print_error("fee_estimates[%d]" % i, fee)
self.notify('fee')
elif method == 'blockchain.relayfee':
if error is None:
self.relay_fee = int(result * COIN)
self.print_error("relayfee", self.relay_fee)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(interface, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(interface, response)
for callback in callbacks:
callback(response)
def get_index(self, method, params):
""" hashable index for subscriptions and cache"""
return str(method) + (':' + str(params[0]) if params else '')
def process_responses(self, interface):
responses = interface.get_responses()
for request, response in responses:
if request:
method, params, message_id = request
k = self.get_index(method, params)
# client requests go through self.send() with a
# callback, are only sent to the current interface,
# and are placed in the unanswered_requests dictionary
client_req = self.unanswered_requests.pop(message_id, None)
if client_req:
assert interface == self.interface
callbacks = [client_req[2]]
else:
# fixme: will only work for subscriptions
k = self.get_index(method, params)
callbacks = self.subscriptions.get(k, [])
# Copy the request method and params to the response
response['method'] = method
response['params'] = params
# Only once we've received a response to an addr subscription
# add it to the list; avoids double-sends on reconnection
if method == 'blockchain.address.subscribe':
self.subscribed_addresses.add(params[0])
else:
if not response: # Closed remotely / misbehaving
self.connection_down(interface.server)
break
# Rewrite response shape to match subscription request response
method = response.get('method')
params = response.get('params')
k = self.get_index(method, params)
if method == 'blockchain.headers.subscribe':
response['result'] = params[0]
response['params'] = []
elif method == 'blockchain.address.subscribe':
response['params'] = [params[0]] # addr
response['result'] = params[1]
callbacks = self.subscriptions.get(k, [])
# update cache if it's a subscription
if method.endswith('.subscribe'):
self.sub_cache[k] = response
# Response is now in canonical form
self.process_response(interface, response, callbacks)
def send(self, messages, callback):
'''Messages is a list of (method, params) tuples'''
with self.lock:
self.pending_sends.append((messages, callback))
def process_pending_sends(self):
# Requests needs connectivity. If we don't have an interface,
# we cannot process them.
if not self.interface:
return
with self.lock:
sends = self.pending_sends
self.pending_sends = []
for messages, callback in sends:
for method, params in messages:
r = None
if method.endswith('.subscribe'):
k = self.get_index(method, params)
# add callback to list
l = self.subscriptions.get(k, [])
if callback not in l:
l.append(callback)
self.subscriptions[k] = l
# check cached response for subscriptions
r = self.sub_cache.get(k)
if r is not None:
util.print_error("cache hit", k)
callback(r)
else:
message_id = self.queue_request(method, params)
self.unanswered_requests[message_id] = method, params, callback
def unsubscribe(self, callback):
'''Unsubscribe a callback to free object references to enable GC.'''
# Note: we can't unsubscribe from the server, so if we receive
# subsequent notifications process_response() will emit a harmless
# "received unexpected notification" warning
with self.lock:
for v in self.subscriptions.values():
if callback in v:
v.remove(callback)
def connection_down(self, server):
'''A connection to server either went down, or was never made.
We distinguish by whether it is in self.interfaces.'''
self.disconnected_servers.add(server)
if server == self.default_server:
self.set_status('disconnected')
if server in self.interfaces:
self.close_interface(self.interfaces[server])
self.headers.pop(server, None)
self.notify('interfaces')
for b in self.blockchains.values():
if b.catch_up == server:
b.catch_up = None
def get_checkpoint(self):
return max(self.blockchains.keys())
def new_interface(self, server, socket):
# todo: get tip first, then decide which checkpoint to use.
self.add_recent_server(server)
interface = Interface(server, socket)
interface.blockchain = None
interface.tip = 0
interface.mode = 'checkpoint'
self.interfaces[server] = interface
self.request_header(interface, self.get_checkpoint())
if server == self.default_server:
self.switch_to_interface(server)
self.notify('interfaces')
def maintain_sockets(self):
'''Socket maintenance.'''
# Responses to connection attempts?
while not self.socket_queue.empty():
server, socket = self.socket_queue.get()
if server in self.connecting:
self.connecting.remove(server)
if socket:
self.new_interface(server, socket)
else:
self.connection_down(server)
# Send pings and shut down stale interfaces
for interface in self.interfaces.values():
if interface.has_timed_out():
self.connection_down(interface.server)
elif interface.ping_required():
params = [ELECTRUM_VERSION, PROTOCOL_VERSION]
self.queue_request('server.version', params, interface)
now = time.time()
# nodes
if len(self.interfaces) + len(self.connecting) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
if not self.is_connecting():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, idx):
interface.print_error("requesting chunk %d" % idx)
self.queue_request('blockchain.block.get_chunk', [idx], interface)
interface.request = idx
interface.req_time = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
error = response.get('error')
result = response.get('result')
params = response.get('params')
if result is None or params is None or error is not None:
interface.print_error(error or 'bad response')
return
# Ignore unsolicited chunks
index = params[0]
if interface.request != index:
return
connect = interface.blockchain.connect_chunk(index, result)
# If not finished, get the next chunk
if not connect:
return
if interface.blockchain.height() < interface.tip:
self.request_chunk(interface, index+1)
else:
interface.request = None
interface.mode = 'default'
interface.print_error('catch up done', interface.blockchain.height())
interface.blockchain.catch_up = None
self.notify('updated')
def request_header(self, interface, height):
#interface.print_error("requesting header %d" % height)
self.queue_request('blockchain.block.get_header', [height], interface)
interface.request = height
interface.req_time = time.time()
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
header = response.get('result')
if not header:
interface.print_error(response)
self.connection_down(interface.server)
return
height = header.get('block_height')
if interface.request != height:
interface.print_error("unsolicited header",interface.request, height)
self.connection_down(interface.server)
return
self.on_header(interface, header)
def can_connect(self, header):
for blockchain in self.blockchains.values():
if blockchain.can_connect(header):
return blockchain
def on_header(self, interface, header):
height = header.get('block_height')
if interface.mode == 'checkpoint':
b = get_blockchain(header)
if b:
interface.mode = 'default'
interface.blockchain = b
self.queue_request('blockchain.headers.subscribe', [], interface)
else:
interface.print_error("checkpoint failed")
self.connection_down(interface.server)
interface.request = None
return
can_connect = interface.blockchain.can_connect(header)
if interface.mode == 'backward':
if can_connect:
interface.good = height
interface.mode = 'binary'
interface.print_error("binary search")
next_height = (interface.bad + interface.good) // 2
else:
if height == 0:
self.connection_down(interface.server)
next_height = None
else:
interface.bad = height
delta = interface.tip - height
next_height = max(0, interface.tip - 2 * delta)
elif interface.mode == 'binary':
if can_connect:
interface.good = height
else:
interface.bad = height
if interface.bad != interface.good + 1:
next_height = (interface.bad + interface.good) // 2
else:
interface.print_error("can connect at %d"% interface.good)
b = self.blockchains.get(interface.good)
if b is None:
b = interface.blockchain.fork(interface.good)
b.catch_up = interface.server
interface.print_error("catching up with new chain")
self.blockchains[interface.good] = b
interface.mode = 'catch_up'
next_height = interface.good
interface.blockchain = b
# todo: garbage collect blockchain objects
self.notify('updated')
elif interface.mode == 'catch_up':
if can_connect:
interface.blockchain.save_header(header)
next_height = height + 1 if height < interface.tip else None
else:
# go back
interface.print_error("cannot connect", height)
interface.mode = 'backward'
interface.bad = height
next_height = height - 1
if next_height is None:
# exit catch_up state
interface.request = None
interface.mode = 'default'
interface.print_error('catch up done', interface.blockchain.height())
interface.blockchain.catch_up = None
self.notify('updated')
elif interface.mode == 'default':
assert not can_connect
interface.print_error("cannot connect %d"% height)
interface.mode = 'backward'
interface.bad = height
# save height where we failed
interface.blockchain_height = interface.blockchain.height()
next_height = height - 1
else:
raise BaseException(interface.mode)
# If not finished, get the next header
if next_height:
if interface.mode == 'catch_up' and interface.tip > next_height + 50:
self.request_chunk(interface, next_height // 2016)
else:
self.request_header(interface, next_height)
# refresh network dialog
self.notify('interfaces')
def maintain_requests(self):
for interface in self.interfaces.values():
if interface.request and time.time() - interface.request_time > 20:
interface.print_error("blockchain request timed out")
self.connection_down(interface.server)
continue
def wait_on_sockets(self):
# Python docs say Windows doesn't like empty selects.
# Sleep to prevent busy looping
if not self.interfaces:
time.sleep(0.1)
return
rin = [i for i in self.interfaces.values()]
win = [i for i in self.interfaces.values() if i.num_requests()]
try:
rout, wout, xout = select.select(rin, win, [], 0.1)
except socket.error as (code, msg):
if code == errno.EINTR:
return
raise
assert not xout
for interface in wout:
interface.send_requests()
for interface in rout:
self.process_responses(interface)
class Network(threading.Thread):
def __init__(self, config={}):
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.callbacks = {}
self.protocol = self.config.get('protocol', 's')
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = [] # returned by interface (list from irc)
self.pending_servers = set([])
self.disconnected_servers = set([])
self.recent_servers = self.config.get('recent_servers',
[]) # successful connections
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.server_lag = 0
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# default subscriptions
self.subscriptions = {}
self.subscriptions[self.on_banner] = [('server.banner', [])]
self.subscriptions[self.on_peers] = [('server.peers.subscribe', [])]
self.pending_transactions_for_notifications = []
def is_connected(self):
return self.interface and self.interface.is_connected
def is_up_to_date(self):
return self.interface.is_up_to_date()
def main_server(self):
return self.interface.server
def send_subscriptions(self):
for cb, sub in self.subscriptions.items():
self.interface.send(sub, cb)
def subscribe(self, messages, callback):
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in messages:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
if self.is_connected():
self.interface.send(messages, callback)
def send(self, messages, callback):
if self.is_connected():
self.interface.send(messages, callback)
return True
else:
return False
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event):
with self.lock:
callbacks = self.callbacks.get(event, [])[:]
if callbacks:
[callback() for callback in callbacks]
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys(
):
continue
else:
choice_list.append(s)
if not choice_list:
if not self.interfaces:
# we are probably offline, retry later
self.disconnected_servers = set([])
return
server = random.choice(choice_list)
return server
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.pending_servers.add(server)
i.start(self.queue)
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, wait=False):
self.start_interfaces()
threading.Thread.start(self)
if wait:
return self.wait_until_connected()
def wait_until_connected(self):
"wait until connection status is known"
if self.config.get('auto_cycle'):
# self.random_server() returns None if all servers have been tried
while not self.is_connected() and self.random_server():
time.sleep(0.1)
else:
self.interface.connect_event.wait()
return self.interface.is_connected
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([host, port, protocol])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values():
i.stop()
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected:
self.switch_to_random_interface()
else:
if self.server_lag > 0:
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
if self.interfaces:
self.switch_to_interface(random.choice(self.interfaces.values()))
def switch_to_interface(self, interface):
assert not self.interface.is_connected
server = interface.server
print_error("switching to", server)
self.interface = interface
h = self.heights.get(server)
if h:
self.server_lag = self.blockchain.height() - h
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.trigger_callback('connected')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected:
self.stop_interface()
# notify gui
self.trigger_callback('disconnecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface(self.interfaces[server])
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0, s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
h = self.heights.get(self.interface.server)
if h:
self.server_lag = blockchain_height - h
if self.server_lag > 1:
print_error("Server is lagging", blockchain_height, h)
if self.config.get('auto_cycle'):
self.set_server(i.server)
else:
print_error('no height for main interface')
self.trigger_callback('updated')
def run(self):
self.blockchain.start()
with self.lock:
self.running = True
while self.is_running():
try:
i = self.queue.get(timeout=30 if self.interfaces else 3)
except Queue.Empty:
if len(self.interfaces) < self.num_server:
self.start_random_interface()
continue
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected:
#if i.server in self.interfaces: raise
self.interfaces[i.server] = i
self.add_recent_server(i)
i.send([('blockchain.headers.subscribe', [])], self.on_header)
if i == self.interface:
print_error('sending subscriptions to',
self.interface.server)
self.send_subscriptions()
self.trigger_callback('connected')
else:
self.disconnected_servers.add(i.server)
if i.server in self.interfaces:
self.interfaces.pop(i.server)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
#self.interface = None
self.trigger_callback('disconnected')
if not self.interface.is_connected and self.config.get(
'auto_cycle'):
self.switch_to_random_interface()
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i, result))
if i == self.interface:
self.server_lag = self.blockchain.height() - height
if self.server_lag > 1 and self.config.get('auto_cycle'):
print_error("Server lagging, stopping interface")
self.stop_interface()
self.trigger_callback('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.trigger_callback('peers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.trigger_callback('banner')
def stop(self):
with self.lock:
self.running = False
def is_running(self):
with self.lock:
return self.running
def synchronous_get(self, requests, timeout=100000000):
return self.interface.synchronous_get(requests)
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote
electrum servers, each connection is handled by its own
thread object returned from Interface(). Its external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None, active_chain=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
if active_chain is None:
active_chain = chainparams.get_active_chain()
self.active_chain = active_chain
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self, self.active_chain)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# run() will block while this event is cleared
self.chain_switched = threading.Event()
self.chain_switched.set()
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.sanitize_default_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def switch_chains(self, chaincode=None):
if chaincode is None:
chain = chainparams.get_active_chain()
else:
chain = chainparams.get_chain_instance(chaincode)
self.chain_switched.clear()
self.active_chain = chain
if self.config.get_active_chain_code() != self.active_chain.code:
self.config.set_active_chain_code(self.active_chain.code)
self.print_error('switching chains to {}'.format(chain.code))
self.stop_network()
time.sleep(0.2)
self.bc_requests.clear()
self.blockchain = Blockchain(self.config, self, self.active_chain)
self.queue = Queue.Queue()
self.sanitize_default_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.interface = None
self.interfaces = {}
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
self.blockchain.init()
# Start the new network
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
time.sleep(0.2)
self.chain_switched.set()
def sanitize_default_server(self):
"""Load the default server from config."""
self.default_server = self.config.get('server')
self.use_ssl = self.config.get('use_ssl', True)
try:
deserialize_server(self.default_server)
except Exception:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server(active_chain=self.active_chain, protocol=get_protocol_letter(self.use_ssl))
def read_recent_servers(self):
return self.config.get('recent_servers', [])
def save_recent_servers(self):
self.config.set_key('recent_servers', self.recent_servers, True)
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
for r in self.unanswered_requests.values():
self.interface.send_request(r)
for addr in self.subscribed_addresses:
self.interface.send_request({'method':'blockchain.address.subscribe','params':[addr]})
self.interface.send_request({'method':'server.banner','params':[]})
self.interface.send_request({'method':'server.peers.subscribe','params':[]})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect()
def auto_connect(self):
return self.config.get('auto_connect', True)
def get_interfaces(self):
'''The interfaces that are in connected state'''
return [s for s, i in self.interfaces.items() if i.is_connected()]
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = self.active_chain.DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if not server in self.interfaces.keys():
if server == self.default_server:
self.set_status('connecting')
i = interface.Interface(server, self.queue, self.config)
self.interfaces[i.server] = i
i.start()
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for i in self.interfaces.values():
i.stop()
self.interface = None
self.interfaces = {}
def set_parameters(self, host, port, protocol, proxy, auto_connect):
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
servers = self.get_interfaces() # Those in connected state
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion = None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect():
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.print_error("starting %s; will switch once connected" % server)
self.start_interface(server)
return
i = self.interfaces[server]
if not i.is_connected():
# do nothing; we will switch once connected
return
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.stop_interface()
self.interface = i
self.addr_responses = {}
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
if self.interface:
self.interface.stop()
self.interface = None
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_if_notification(self, i):
'''Handle interface addition and removal through notifications'''
if i.is_connected():
self.add_recent_server(i)
i.send_request({'method':'blockchain.headers.subscribe','params':[]})
if i.server == self.default_server:
self.switch_to_interface(i.server)
else:
self.interfaces.pop(i.server, None)
self.heights.pop(i.server, None)
if i == self.interface:
self.interface = None
self.addr_responses = {}
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
# Our set of interfaces changed
self.notify('interfaces')
def process_response(self, i, response):
# the id comes from the daemon or the network proxy
_id = response.get('id')
if _id is not None:
if i != self.interface:
return
self.unanswered_requests.pop(_id)
method = response.get('method')
result = response.get('result')
if method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
self.banner = result
self.notify('banner')
elif method == 'blockchain.address.subscribe':
addr = response.get('params')[0]
self.addr_responses[addr] = result
self.response_queue.put(response)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(i, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(i, response)
else:
self.response_queue.put(response)
def handle_requests(self):
'''Some requests require connectivity, others we handle locally in
process_request() and must do so in order to e.g. prevent the
daemon seeming unresponsive.
'''
unhandled = []
while not self.requests_queue.empty():
request = self.requests_queue.get()
if not self.process_request(request):
unhandled.append(request)
for request in unhandled:
self.requests_queue.put(request)
def process_request(self, request):
'''Returns true if the request was processed.'''
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
out['result'] = f(*params)
except AttributeError:
out['error'] = "unknown method"
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return True
if method == 'blockchain.address.subscribe':
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({'id':_id, 'result':self.addr_responses[addr]})
return True
# This request needs connectivity. If we don't have an
# interface, we cannot process it.
if not self.is_connected():
return False
self.unanswered_requests[_id] = request
self.interface.send_request(request)
return True
def check_interfaces(self):
now = time.time()
# nodes
if len(self.interfaces) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('retrying connections to reach preferred number of interfaces ({}/{})'.format(len(self.interfaces), self.num_server))
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
interface.send_request({'method':'blockchain.block.get_chunk', 'params':[idx]})
data['chunk_idx'] = idx
data['req_time'] = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx, response['result'])
# If not finished, get the next chunk
if idx < 0 or not idx or self.get_local_height() >= data['if_height']:
self.bc_requests.popleft()
if not idx:
interface.print_error("header didn't match checkpoint, dismissing interface")
interface.stop()
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
interface.send_request({'method':'blockchain.block.get_header', 'params':[height]})
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(data['chain'], response['result'])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify('updated')
else:
interface.print_error("header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them, if necessary'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) / self.active_chain.chunk_size)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface.is_connected():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
interface.stop()
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def run(self):
self.blockchain.init()
while self.is_running():
# wait if we're switching chains
self.chain_switched.wait()
self.check_interfaces()
self.handle_requests()
self.handle_bc_requests()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
# if response is None it is a notification about the interface
if response is None:
self.process_if_notification(i)
else:
self.process_response(i, response)
self.stop_network()
self.print_error("stopped")
def on_header(self, i, r):
header = r.get('result')
if not header:
return
height = header.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get('merkle_root')
self.utxo_roots[i.server] = header.get('utxo_root')
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {'if_height': height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class NetworkProxy(util.DaemonThread):
def __init__(self, socket, config=None):
if config is None:
config = {} # Do not use mutables as default arguments!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.message_id = 0
self.unanswered_requests = {}
self.subscriptions = {}
self.debug = False
self.lock = threading.Lock()
self.callbacks = {}
if socket:
self.pipe = util.SocketPipe(socket)
self.network = None
else:
self.pipe = util.QueuePipe()
self.network = Network(self.pipe, config)
self.network.start()
for key in ["fee", "status", "banner", "updated", "servers", "interfaces"]:
value = self.network.get_status_value(key)
self.pipe.get_queue.put({"method": "network.status", "params": [key, value]})
# status variables
self.status = "unknown"
self.servers = {}
self.banner = ""
self.blockchain_height = 0
self.server_height = 0
self.interfaces = []
self.jobs = []
# value returned by estimatefee
self.fee = None
def run(self):
while self.is_running():
for job in self.jobs:
job()
try:
response = self.pipe.get()
except util.timeout:
continue
if response is None:
break
self.process(response)
self.trigger_callback("stop")
if self.network:
self.network.stop()
self.print_error("stopped")
def process(self, response):
if self.debug:
print_error("<--", response)
if response.get("method") == "network.status":
key, value = response.get("params")
if key == "status":
self.status = value
elif key == "banner":
self.banner = value
elif key == "fee":
self.fee = value
elif key == "updated":
self.blockchain_height, self.server_height = value
elif key == "servers":
self.servers = value
elif key == "interfaces":
self.interfaces = value
if key in ["status", "updated"]:
self.trigger_callback(key)
else:
self.trigger_callback(key, (value,))
return
msg_id = response.get("id")
result = response.get("result")
error = response.get("error")
if msg_id is not None:
with self.lock:
method, params, callback = self.unanswered_requests.pop(msg_id)
else:
method = response.get("method")
params = response.get("params")
with self.lock:
for k, v in self.subscriptions.items():
if (method, params) in v:
callback = k
break
else:
print_error("received unexpected notification", method, params)
return
r = {"method": method, "params": params, "result": result, "id": msg_id, "error": error}
callback(r)
def send(self, messages, callback):
"""return the ids of the requests that we sent"""
# detect subscriptions
sub = []
for message in messages:
m, v = message
if m[-10:] == ".subscribe":
sub.append(message)
if sub:
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in sub:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
with self.lock:
requests = []
ids = []
for m in messages:
method, params = m
request = {"id": self.message_id, "method": method, "params": params}
self.unanswered_requests[self.message_id] = method, params, callback
ids.append(self.message_id)
requests.append(request)
if self.debug:
print_error("-->", request)
self.message_id += 1
self.pipe.send_all(requests)
return ids
def synchronous_get(self, requests, timeout=100000000):
queue = Queue.Queue()
ids = self.send(requests, queue.put)
id2 = ids[:]
res = {}
while ids:
r = queue.get(True, timeout)
_id = r.get("id")
ids.remove(_id)
if r.get("error"):
raise BaseException(r.get("error"))
result = r.get("result")
res[_id] = r.get("result")
out = []
for _id in id2:
out.append(res[_id])
return out
def get_servers(self):
return self.servers
def get_interfaces(self):
return self.interfaces
def get_header(self, height):
return self.synchronous_get([("network.get_header", [height])])[0]
def get_local_height(self):
return self.blockchain_height
def get_server_height(self):
return self.server_height
def is_connected(self):
return self.status == "connected"
def is_connecting(self):
return self.status == "connecting"
def is_up_to_date(self):
return self.unanswered_requests == {}
def get_parameters(self):
return self.synchronous_get([("network.get_parameters", [])])[0]
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server_str = serialize_server(host, port, protocol)
self.config.set_key("auto_connect", auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server_str, True)
# abort if changes were not allowed by config
if self.config.get("server") != server_str or self.config.get("proxy") != proxy_str:
return
return self.synchronous_get([("network.set_parameters", (host, port, protocol, proxy, auto_connect))])[0]
def stop_daemon(self):
return self.send([("daemon.stop", [])], None)
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event, params=()):
with self.lock:
callbacks = self.callbacks.get(event, [])[:]
if callbacks:
[callback(*params) for callback in callbacks]
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote electrum
servers, each connected socket is handled by an Interface() object.
Connections are initiated by a Connection() thread which stops once
the connection succeeds or fails.
Our external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = {}
self.socket_queue = Queue.Queue()
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface is not None
def queue_request(self, method, params):
self.interface.queue_request({'method': method, 'params': params})
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
for r in self.unanswered_requests.values():
self.interface.queue_request(r)
for addr in self.subscribed_addresses:
self.queue_request('blockchain.address.subscribe', [addr])
self.queue_request('server.banner', [])
self.queue_request('server.peers.subscribe', [])
self.queue_request('blockchain.estimatefee', [2])
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'fee':
value = self.fee
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect
def get_interfaces(self):
'''The interfaces that are in connected state'''
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if (not server in self.interfaces and not server in self.connecting):
if server == self.default_server:
self.set_status('connecting')
c = Connection(server, self.socket_queue, self.config.path)
self.connecting[server] = c
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
assert not self.connecting and self.socket_queue.empty()
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for interface in self.interfaces.values():
self.close_interface(interface)
assert self.interface is None
assert not self.interfaces
self.connecting = {}
# Get a new queue - no old pending connections thanks!
self.socket_queue = Queue.Queue()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.auto_connect = auto_connect
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
'''Switch to a random connected server other than the current one'''
servers = self.get_interfaces() # Those in connected state
if self.default_server in servers:
servers.remove(self.default_server)
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion = None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.print_error("starting %s; will switch once connected" % server)
self.start_interface(server)
return
i = self.interfaces[server]
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.close_interface(self.interface)
self.interface = i
self.addr_responses = {}
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def close_interface(self, interface):
if interface:
self.interfaces.pop(interface.server)
if interface.server == self.default_server:
self.interface = None
interface.close()
def add_recent_server(self, server):
# list is ordered
if server in self.recent_servers:
self.recent_servers.remove(server)
self.recent_servers.insert(0, server)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_response(self, interface, response):
error = response.get('error')
result = response.get('result')
method = response.get('method')
# We handle some responses; return the rest to the client.
if method == 'server.version':
interface.server_version = result
elif method == 'blockchain.headers.subscribe':
if error is None:
self.on_header(interface, result)
elif method == 'server.peers.subscribe':
if error is None:
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
if error is None:
self.banner = result
self.notify('banner')
elif method == 'blockchain.estimatefee':
if error is None:
self.fee = int(result * COIN)
self.print_error("recommended fee", self.fee)
self.notify('fee')
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(interface, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(interface, response)
else:
# Cache address subscription results
if method == 'blockchain.address.subscribe' and error is None:
addr = response['params'][0]
self.addr_responses[addr] = result
self.response_queue.put(response)
def process_responses(self, interface):
notifications, responses = interface.get_responses()
for request, response in responses:
# Client ID was given by the daemon or proxy
client_id = request.get('id')
if client_id is not None:
if interface != self.interface:
continue
self.unanswered_requests.pop(client_id)
# Copy the request method and params to the response
response['method'] = request.get('method')
response['params'] = request.get('params')
response['id'] = client_id
self.process_response(interface, response)
for response in notifications:
if not response: # Closed remotely
self.connection_down(interface.server)
break
# Rewrite response shape to match subscription request response
method = response.get('method')
if method == 'blockchain.headers.subscribe':
response['result'] = response['params'][0]
response['params'] = []
elif method == 'blockchain.address.subscribe':
params = response['params']
response['params'] = [params[0]] # addr
response['result'] = params[1]
self.process_response(interface, response)
def handle_incoming_requests(self):
while not self.requests_queue.empty():
self.process_request(self.requests_queue.get())
def process_request(self, request):
'''Returns true if the request was processed.'''
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
out['result'] = f(*params)
except AttributeError:
out['error'] = "unknown method"
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return True
if method == 'blockchain.address.subscribe':
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({'id':_id, 'result':self.addr_responses[addr]})
return True
# This request needs connectivity. If we don't have an
# interface, we cannot process it.
if not self.interface:
return False
self.unanswered_requests[_id] = request
self.interface.queue_request(request)
return True
def connection_down(self, server):
'''A connection to server either went down, or was never made.
We distinguish by whether it is in self.interfaces.'''
self.disconnected_servers.add(server)
if server == self.default_server:
self.set_status('disconnected')
if server in self.interfaces:
self.close_interface(self.interfaces[server])
self.heights.pop(server, None)
self.notify('interfaces')
def new_interface(self, server, socket):
self.add_recent_server(server)
self.interfaces[server] = interface = Interface(server, socket)
interface.queue_request({'method': 'blockchain.headers.subscribe',
'params': []})
if server == self.default_server:
self.switch_to_interface(server)
self.notify('interfaces')
def maintain_sockets(self):
'''Socket maintenance.'''
# Responses to connection attempts?
while not self.socket_queue.empty():
server, socket = self.socket_queue.get()
self.connecting.pop(server)
if socket:
self.new_interface(server, socket)
else:
self.connection_down(server)
# Send pings and shut down stale interfaces
for interface in self.interfaces.values():
if interface.has_timed_out():
self.connection_down(interface.server)
elif interface.ping_required():
version_req = {'method': 'server.version',
'params': [ELECTRUM_VERSION, PROTOCOL_VERSION]}
interface.queue_request(version_req)
now = time.time()
# nodes
if len(self.interfaces) + len(self.connecting) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
interface.queue_request({'method':'blockchain.block.get_chunk',
'params':[idx]})
data['chunk_idx'] = idx
data['req_time'] = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx, response['result'])
# If not finished, get the next chunk
if idx < 0 or self.get_local_height() >= data['if_height']:
self.bc_requests.popleft()
self.notify('updated')
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
interface.queue_request({'method':'blockchain.block.get_header',
'params':[height]})
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(data['chain'], response['result'])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify('updated')
else:
interface.print_error("header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them,
if necessary.
'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) / 2016)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object.
'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface in self.interfaces.values():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
self.connection_down(interface.server)
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def wait_on_sockets(self):
# Python docs say Windows doesn't like empty selects.
# Sleep to prevent busy looping
if not self.interfaces:
time.sleep(0.1)
return
rin = [i for i in self.interfaces.values()]
win = [i for i in self.interfaces.values() if i.unsent_requests]
rout, wout, xout = select.select(rin, win, [], 0.1)
assert not xout
for interface in wout:
interface.send_requests()
for interface in rout:
self.process_responses(interface)
def run(self):
self.blockchain.init()
while self.is_running():
self.maintain_sockets()
self.wait_on_sockets()
self.handle_incoming_requests()
self.handle_bc_requests()
self.stop_network()
self.print_error("stopped")
def on_header(self, i, header):
height = header.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get('merkle_root')
self.utxo_roots[i.server] = header.get('utxo_root')
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {'if_height': height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class NetworkProxy(util.DaemonThread):
def __init__(self, socket, config=None):
if config is None:
config = {} # Do not use mutables as default arguments!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.message_id = 0
self.unanswered_requests = {}
self.subscriptions = {}
self.debug = False
self.lock = threading.Lock()
self.callbacks = {}
if socket:
self.pipe = util.SocketPipe(socket)
self.network = None
else:
self.pipe = util.QueuePipe()
self.network = Network(self.pipe, config)
self.network.start()
for key in ['fee','status','banner','updated','servers','interfaces']:
value = self.network.get_status_value(key)
self.pipe.get_queue.put({'method':'network.status', 'params':[key, value]})
# status variables
self.status = 'unknown'
self.servers = {}
self.banner = ''
self.blockchain_height = 0
self.server_height = 0
self.interfaces = []
# value returned by estimatefee
self.fee = None
def run(self):
while self.is_running():
self.run_jobs() # Synchronizer and Verifier
try:
response = self.pipe.get()
except util.timeout:
continue
if response is None:
break
# Protect against ill-formed or malicious server responses
try:
self.process(response)
except:
traceback.print_exc(file=sys.stderr)
self.trigger_callback('stop')
if self.network:
self.network.stop()
self.print_error("stopped")
def process(self, response):
if self.debug:
self.print_error("<--", response)
if response.get('method') == 'network.status':
key, value = response.get('params')
if key == 'status':
self.status = value
elif key == 'banner':
self.banner = value
elif key == 'fee':
self.fee = value
elif key == 'updated':
self.blockchain_height, self.server_height = value
elif key == 'servers':
self.servers = value
elif key == 'interfaces':
self.interfaces = value
if key in ['status', 'updated']:
self.trigger_callback(key)
else:
self.trigger_callback(key, (value,))
return
msg_id = response.get('id')
result = response.get('result')
error = response.get('error')
if msg_id is not None:
with self.lock:
method, params, callback = self.unanswered_requests.pop(msg_id)
else:
method = response.get('method')
params = response.get('params')
with self.lock:
for k,v in self.subscriptions.items():
if (method, params) in v:
callback = k
break
else:
self.print_error("received unexpected notification",
method, params)
return
r = {'method':method, 'params':params, 'result':result,
'id':msg_id, 'error':error}
callback(r)
def send(self, messages, callback):
"""return the ids of the requests that we sent"""
# detect subscriptions
sub = []
for message in messages:
m, v = message
if m[-10:] == '.subscribe':
sub.append(message)
if sub:
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in sub:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
with self.lock:
requests = []
ids = []
for m in messages:
method, params = m
request = { 'id':self.message_id, 'method':method, 'params':params }
self.unanswered_requests[self.message_id] = method, params, callback
ids.append(self.message_id)
requests.append(request)
if self.debug:
self.print_error("-->", request)
self.message_id += 1
self.pipe.send_all(requests)
return ids
def synchronous_get(self, requests, timeout=100000000):
queue = Queue.Queue()
ids = self.send(requests, queue.put)
id2 = ids[:]
res = {}
while ids:
r = queue.get(True, timeout)
_id = r.get('id')
ids.remove(_id)
if r.get('error'):
raise BaseException(r.get('error'))
result = r.get('result')
res[_id] = r.get('result')
out = []
for _id in id2:
out.append(res[_id])
return out
def get_servers(self):
return self.servers
def get_interfaces(self):
return self.interfaces
def get_local_height(self):
return self.blockchain_height
def get_server_height(self):
return self.server_height
def is_connected(self):
return self.status == 'connected'
def is_connecting(self):
return self.status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def get_parameters(self):
return self.synchronous_get([('network.get_parameters', [])])[0]
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server_str = serialize_server(host, port, protocol)
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server_str, True)
# abort if changes were not allowed by config
if self.config.get('server') != server_str or self.config.get('proxy') != proxy_str:
return
return self.synchronous_get([('network.set_parameters', (host, port, protocol, proxy, auto_connect))])[0]
def stop_daemon(self):
return self.send([('daemon.stop',[])], None)
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event, params=()):
with self.lock:
callbacks = self.callbacks.get(event,[])[:]
if callbacks:
[callback(*params) for callback in callbacks]
class NetworkProxy(threading.Thread):
# connects to daemon
# sends requests, runs callbacks
def __init__(self, config = {}):
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.daemon_port = config.get('daemon_port', 8000)
self.message_id = 0
self.unanswered_requests = {}
self.subscriptions = {}
self.debug = False
self.lock = threading.Lock()
self.pending_transactions_for_notifications = []
def start(self, start_daemon=False):
daemon_started = False
while True:
try:
self.socket.connect(('', self.daemon_port))
threading.Thread.start(self)
return True
except socket.error:
if not start_daemon:
return False
elif not daemon_started:
print_stderr( "Starting daemon [%s]"%self.config.get('server'))
daemon_started = True
pid = os.fork()
if (pid == 0): # The first child.
os.chdir("/")
os.setsid()
os.umask(0)
pid2 = os.fork()
if (pid2 == 0): # Second child
server = NetworkServer(self.config)
try:
server.main_loop()
except KeyboardInterrupt:
print "Ctrl C - Stopping server"
sys.exit(1)
sys.exit(0)
else:
time.sleep(0.1)
def parse_json(self, message):
s = message.find('\n')
if s==-1:
return None, message
j = json.loads( message[0:s] )
return j, message[s+1:]
def run(self):
# read responses and trigger callbacks
message = ''
while True:
try:
data = self.socket.recv(1024)
except:
data = ''
if not data:
break
message += data
while True:
response, message = self.parse_json(message)
if response is not None:
self.process(response)
else:
break
print "NetworkProxy: exiting"
def process(self, response):
# runs callbacks
if self.debug: print "<--", response
msg_id = response.get('id')
with self.lock:
method, params, callback = self.unanswered_requests.pop(msg_id)
result = response.get('result')
callback(None, {'method':method, 'params':params, 'result':result, 'id':msg_id})
def subscribe(self, messages, callback):
# detect if it is a subscription
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in messages:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
self.send( messages, callback )
def send(self, messages, callback):
"""return the ids of the requests that we sent"""
out = ''
ids = []
for m in messages:
method, params = m
request = json.dumps( { 'id':self.message_id, 'method':method, 'params':params } )
self.unanswered_requests[self.message_id] = method, params, callback
ids.append(self.message_id)
if self.debug: print "-->", request
self.message_id += 1
out += request + '\n'
while out:
sent = self.socket.send( out )
out = out[sent:]
return ids
def synchronous_get(self, requests, timeout=100000000):
queue = Queue.Queue()
ids = self.send(requests, lambda i,x: queue.put(x))
id2 = ids[:]
res = {}
while ids:
r = queue.get(True, timeout)
_id = r.get('id')
if _id in ids:
ids.remove(_id)
res[_id] = r.get('result')
out = []
for _id in id2:
out.append(res[_id])
return out
def get_servers(self):
return self.synchronous_get([('network.get_servers',[])])[0]
def get_header(self, height):
return self.synchronous_get([('network.get_header',[height])])[0]
def get_local_height(self):
return self.synchronous_get([('network.get_local_height',[])])[0]
def is_connected(self):
return self.synchronous_get([('network.is_connected',[])])[0]
def is_up_to_date(self):
return self.synchronous_get([('network.is_up_to_date',[])])[0]
def main_server(self):
return self.synchronous_get([('network.main_server',[])])[0]
def stop(self):
return self.synchronous_get([('daemon.shutdown',[])])[0]
def trigger_callback(self, cb):
pass
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote electrum
servers, each connected socket is handled by an Interface() object.
Connections are initiated by a Connection() thread which stops once
the connection succeeds or fails.
Our external API:
- Member functions get_header(), get_interfaces(), get_local_height(),
get_parameters(), get_server_height(), get_status_value(),
is_connected(), new_blockchain_height(), set_parameters(),
stop()
"""
def __init__(self, config=None, plugins=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.lock = Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.subscriptions = defaultdict(list)
self.callbacks = defaultdict(list)
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
self.plugins = plugins
if self.plugins:
self.plugins.set_network(self)
def register_callback(self, event, callback):
with self.lock:
self.callbacks[event].append(callback)
def trigger_callback(self, event, *args):
with self.lock:
callbacks = self.callbacks[event][:]
[callback(*args) for callback in callbacks]
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface is not None
def is_connecting(self):
return self.connection_status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def queue_request(self, method, params, interface=None):
# If you want to queue a request on any interface it must go
# through this function so message ids are properly tracked
if interface is None:
interface = self.interface
message_id = self.message_id
self.message_id += 1
if self.debug:
self.print_error(interface.host, "-->", method, params, message_id)
interface.queue_request(method, params, message_id)
return message_id
def send_subscriptions(self):
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
# Resend unanswered requests
requests = self.unanswered_requests.values()
self.unanswered_requests = {}
for request in requests:
message_id = self.queue_request(request[0], request[1])
self.unanswered_requests[message_id] = request
for addr in self.subscribed_addresses:
self.queue_request('blockchain.address.subscribe', [addr])
self.queue_request('server.banner', [])
self.queue_request('server.peers.subscribe', [])
self.queue_request('blockchain.estimatefee', [2])
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'fee':
value = self.fee
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
if key in ['status', 'updated']:
self.trigger_callback(key)
else:
self.trigger_callback(key, self.get_status_value(key))
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect
def get_interfaces(self):
'''The interfaces that are in connected state'''
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if (not server in self.interfaces and not server in self.connecting):
if server == self.default_server:
self.print_error("connecting to %s as new interface" % server)
self.set_status('connecting')
self.connecting.add(server)
c = Connection(server, self.socket_queue, self.config.path)
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
assert not self.connecting and self.socket_queue.empty()
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for interface in self.interfaces.values():
self.close_interface(interface)
assert self.interface is None
assert not self.interfaces
self.connecting = set()
# Get a new queue - no old pending connections thanks!
self.socket_queue = Queue.Queue()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server = serialize_server(host, port, protocol)
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server, True)
# abort if changes were not allowed by config
if self.config.get('server') != server or self.config.get('proxy') != proxy_str:
return
self.auto_connect = auto_connect
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
'''Switch to a random connected server other than the current one'''
servers = self.get_interfaces() # Those in connected state
if self.default_server in servers:
servers.remove(self.default_server)
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion = None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.interface = None
self.start_interface(server)
return
i = self.interfaces[server]
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.close_interface(self.interface)
self.interface = i
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def close_interface(self, interface):
if interface:
self.interfaces.pop(interface.server)
if interface.server == self.default_server:
self.interface = None
interface.close()
def add_recent_server(self, server):
# list is ordered
if server in self.recent_servers:
self.recent_servers.remove(server)
self.recent_servers.insert(0, server)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_response(self, interface, response, callback):
if self.debug:
self.print_error("<--", response)
error = response.get('error')
result = response.get('result')
method = response.get('method')
# We handle some responses; return the rest to the client.
if method == 'server.version':
interface.server_version = result
elif method == 'blockchain.headers.subscribe':
if error is None:
self.on_header(interface, result)
elif method == 'server.peers.subscribe':
if error is None:
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
if error is None:
self.banner = result
self.notify('banner')
elif method == 'blockchain.estimatefee':
if error is None:
self.fee = int(result * COIN)
self.print_error("recommended fee", self.fee)
self.notify('fee')
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(interface, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(interface, response)
else:
if callback is None:
params = response['params']
with self.lock:
for k,v in self.subscriptions.items():
if (method, params) in v:
callback = k
break
if callback is None:
self.print_error("received unexpected notification",
method, params)
else:
callback(response)
def process_responses(self, interface):
responses = interface.get_responses()
for request, response in responses:
callback = None
if request:
method, params, message_id = request
# client requests go through self.send() with a
# callback, are only sent to the current interface,
# and are placed in the unanswered_requests dictionary
client_req = self.unanswered_requests.pop(message_id, None)
if client_req:
assert interface == self.interface
callback = client_req[2]
# Copy the request method and params to the response
response['method'] = method
response['params'] = params
# Only once we've received a response to an addr subscription
# add it to the list; avoids double-sends on reconnection
if method == 'blockchain.address.subscribe':
self.subscribed_addresses.add(params[0])
else:
if not response: # Closed remotely / misbehaving
self.connection_down(interface.server)
break
# Rewrite response shape to match subscription request response
method = response.get('method')
params = response.get('params')
if method == 'blockchain.headers.subscribe':
response['result'] = params[0]
response['params'] = []
elif method == 'blockchain.address.subscribe':
response['params'] = [params[0]] # addr
response['result'] = params[1]
# Response is now in canonical form
self.process_response(interface, response, callback)
def send(self, messages, callback):
'''Messages is a list of (method, value) tuples'''
with self.lock:
self.pending_sends.append((messages, callback))
def process_pending_sends(self):
# Requests needs connectivity. If we don't have an interface,
# we cannot process them.
if not self.interface:
return
with self.lock:
sends = self.pending_sends
self.pending_sends = []
for messages, callback in sends:
subs = filter(lambda (m,v): m.endswith('.subscribe'), messages)
with self.lock:
for sub in subs:
if sub not in self.subscriptions[callback]:
self.subscriptions[callback].append(sub)
for method, params in messages:
message_id = self.queue_request(method, params)
self.unanswered_requests[message_id] = method, params, callback
def unsubscribe(self, callback):
'''Unsubscribe a callback to free object references to enable GC.'''
# Note: we can't unsubscribe from the server, so if we receive
# subsequent notifications process_response() will emit a harmless
# "received unexpected notification" warning
self.subscriptions.pop(callback, None)
def connection_down(self, server):
'''A connection to server either went down, or was never made.
We distinguish by whether it is in self.interfaces.'''
self.disconnected_servers.add(server)
if server == self.default_server:
self.set_status('disconnected')
if server in self.interfaces:
self.close_interface(self.interfaces[server])
self.heights.pop(server, None)
self.notify('interfaces')
def new_interface(self, server, socket):
self.add_recent_server(server)
self.interfaces[server] = interface = Interface(server, socket)
self.queue_request('blockchain.headers.subscribe', [], interface)
if server == self.default_server:
self.switch_to_interface(server)
self.notify('interfaces')
def maintain_sockets(self):
'''Socket maintenance.'''
# Responses to connection attempts?
while not self.socket_queue.empty():
server, socket = self.socket_queue.get()
self.connecting.remove(server)
if socket:
self.new_interface(server, socket)
else:
self.connection_down(server)
# Send pings and shut down stale interfaces
for interface in self.interfaces.values():
if interface.has_timed_out():
self.connection_down(interface.server)
elif interface.ping_required():
params = [ELECTRUM_VERSION, PROTOCOL_VERSION]
self.queue_request('server.version', params, interface)
now = time.time()
# nodes
if len(self.interfaces) + len(self.connecting) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
if not self.is_connecting():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
self.queue_request('blockchain.block.get_chunk', [idx], interface)
data['chunk_idx'] = idx
data['req_time'] = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx, response['result'])
# If not finished, get the next chunk
if idx < 0 or self.get_local_height() >= data['if_height']:
self.bc_requests.popleft()
self.notify('updated')
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
self.queue_request('blockchain.block.get_header', [height], interface)
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(data['chain'], response['result'])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify('updated')
else:
interface.print_error("header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them,
if necessary.
'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) / 2016)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object.
'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface in self.interfaces.values():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
self.connection_down(interface.server)
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def wait_on_sockets(self):
# Python docs say Windows doesn't like empty selects.
# Sleep to prevent busy looping
if not self.interfaces:
time.sleep(0.1)
return
rin = [i for i in self.interfaces.values()]
win = [i for i in self.interfaces.values() if i.unsent_requests]
rout, wout, xout = select.select(rin, win, [], 0.1)
assert not xout
for interface in wout:
interface.send_requests()
for interface in rout:
self.process_responses(interface)
def run(self):
self.blockchain.init()
while self.is_running():
self.maintain_sockets()
self.wait_on_sockets()
self.handle_bc_requests()
self.run_jobs() # Synchronizer and Verifier
self.process_pending_sends()
self.stop_network()
if self.plugins:
self.plugins.set_network(None)
self.print_error("stopped")
def on_header(self, i, header):
height = header.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get('merkle_root')
self.utxo_roots[i.server] = header.get('utxo_root')
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {'if_height': height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
def synchronous_get(self, request, timeout=100000000):
queue = Queue.Queue()
self.send([request], queue.put)
r = queue.get(True, timeout)
if r.get('error'):
raise BaseException(r.get('error'))
return r.get('result')
class Network(util.DaemonThread):
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server('s')
self.protocol = deserialize_server(self.default_server)[2]
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.recent_servers = self.read_recent_servers()
self.pending_servers = set()
self.banner = ''
self.interface = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# address subscriptions and cached results
self.addresses = {}
self.connection_status = 'connecting'
self.requests_queue = Queue.Queue()
self.set_proxy(deserialize_proxy(self.config.get('proxy')))
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
h = self.get_server_height()
if not h:
self.print_error('no height for main interface')
return False
lag = self.get_local_height() - self.get_server_height()
return lag > 1
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected
def send_subscriptions(self):
for addr in self.addresses:
self.interface.send_request({
'method': 'blockchain.address.subscribe',
'params': [addr]
})
self.interface.send_request({'method': 'server.banner', 'params': []})
self.interface.send_request({
'method': 'server.peers.subscribe',
'params': []
})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({
'method': 'network.status',
'params': [key, value]
})
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys(
):
continue
else:
choice_list.append(s)
if not choice_list:
return
server = random.choice(choice_list)
return server
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
auto_connect = self.config.get('auto_cycle', True)
return host, port, protocol, self.proxy, auto_connect
def get_interfaces(self):
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.pending_servers.add(server)
i.start(self.queue)
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, response_queue):
self.running = True
self.response_queue = response_queue
self.start_interfaces()
t = threading.Thread(target=self.process_requests_thread)
t.start()
self.blockchain.start()
util.DaemonThread.start(self)
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"],
int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(
socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def set_parameters(self, host, port, protocol, proxy, auto_connect):
if self.proxy != proxy or self.protocol != protocol:
self.print_error('restarting network')
for i in self.interfaces.values():
i.stop()
self.interfaces.pop(i.server)
self.set_proxy(proxy)
self.protocol = protocol
self.disconnected_servers = set([])
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected:
self.switch_to_random_interface()
else:
if self.server_is_lagging():
self.stop_interface()
else:
server_str = serialize_server(host, port, protocol)
self.set_server(server_str)
def switch_to_random_interface(self):
while self.interfaces:
i = random.choice(self.interfaces.values())
if i.is_connected:
self.switch_to_interface(i)
break
else:
self.remove_interface(i)
def switch_to_interface(self, interface):
server = interface.server
self.print_error("switching to", server)
self.interface = interface
self.default_server = server
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected:
return
if self.protocol != deserialize_server(server)[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected:
self.stop_interface()
# notify gui
self.set_status('connecting')
# start interface
self.default_server = server
if server in self.interfaces.keys():
self.switch_to_interface(self.interfaces[server])
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0, s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def add_interface(self, i):
self.interfaces[i.server] = i
self.notify('interfaces')
def remove_interface(self, i):
self.interfaces.pop(i.server)
self.notify('interfaces')
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
if self.server_is_lagging():
self.print_error("Server is lagging", blockchain_height,
self.get_server_height())
if self.config.get('auto_cycle'):
self.set_server(i.server)
self.notify('updated')
def process_response(self, i, response):
method = response['method']
if method == 'blockchain.address.subscribe':
self.on_address(i, response)
elif method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.on_peers(i, response)
elif method == 'server.banner':
self.on_banner(i, response)
else:
self.response_queue.put(response)
def process_requests_thread(self):
while self.is_running():
try:
request = self.requests_queue.get(timeout=0.1)
except Queue.Empty:
continue
self.process_request(request)
def process_request(self, request):
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id': _id}
try:
f = getattr(self, method[8:])
except AttributeError:
out['error'] = "unknown method"
try:
out['result'] = f(*params)
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return
if method == 'blockchain.address.subscribe':
addr = params[0]
if addr in self.addresses:
self.response_queue.put({
'id': _id,
'result': self.addresses[addr]
})
return
try:
self.interface.send_request(request)
except:
# put it back in the queue
self.print_error("warning: interface not ready for", request)
self.requests_queue.put(request)
time.sleep(0.1)
def run(self):
server_retry_time = time.time()
nodes_retry_time = time.time()
while self.is_running():
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
now = time.time()
if len(self.interfaces) + len(
self.pending_servers) < self.num_server:
self.start_random_interface()
if not self.interfaces:
if now - nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
nodes_retry_time = now
if not self.interface.is_connected:
if self.config.get('auto_cycle'):
if self.interfaces:
self.switch_to_random_interface()
else:
if self.default_server in self.interfaces.keys():
self.switch_to_interface(
self.interfaces[self.default_server])
else:
if self.default_server in self.disconnected_servers:
if now - server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(
self.default_server)
server_retry_time = now
else:
if self.default_server not in self.pending_servers:
self.print_error("forcing reconnection")
self.interface = self.start_interface(
self.default_server)
continue
if response is not None:
self.process_response(i, response)
continue
# if response is None it is a notification about the interface
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected:
self.add_interface(i)
self.add_recent_server(i)
i.send_request({
'method': 'blockchain.headers.subscribe',
'params': []
})
if i == self.interface:
self.print_error('sending subscriptions to',
self.interface.server)
self.send_subscriptions()
self.set_status('connected')
else:
if i.server in self.interfaces:
self.remove_interface(i)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
self.print_error("stopping interfaces")
for i in self.interfaces.values():
i.stop()
self.print_error("stopped")
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i, result))
if i == self.interface:
if self.server_is_lagging() and self.config.get('auto_cycle'):
self.print_error("Server lagging, stopping interface")
self.stop_interface()
self.notify('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.notify('servers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.notify('banner')
def on_address(self, i, r):
addr = r.get('params')[0]
result = r.get('result')
self.addresses[addr] = result
self.response_queue.put(r)
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class NetworkProxy(util.DaemonThread):
def __init__(self, socket, config=None):
if config is None:
config = {} # Do not use mutables as default arguments!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.message_id = 0
self.unanswered_requests = {}
self.subscriptions = {}
self.debug = False
self.lock = threading.Lock()
self.pending_transactions_for_notifications = []
self.callbacks = {}
if socket:
self.pipe = util.SocketPipe(socket)
self.network = None
else:
self.pipe = util.QueuePipe()
self.network = Network(self.pipe, config)
self.network.start()
for key in [
'status', 'banner', 'updated', 'servers', 'interfaces'
]:
value = self.network.get_status_value(key)
self.pipe.get_queue.put({
'method': 'network.status',
'params': [key, value]
})
# status variables
self.status = 'connecting'
self.servers = {}
self.banner = ''
self.blockchain_height = 0
self.server_height = 0
self.interfaces = []
self.jobs = []
def run(self):
while self.is_running():
for job in self.jobs:
job()
try:
response = self.pipe.get()
except util.timeout:
continue
if response is None:
break
self.process(response)
self.trigger_callback('stop')
if self.network:
self.network.stop()
self.print_error("stopped")
def process(self, response):
if self.debug:
print_error("<--", response)
if response.get('method') == 'network.status':
key, value = response.get('params')
if key == 'status':
self.status = value
elif key == 'banner':
self.banner = value
elif key == 'updated':
self.blockchain_height, self.server_height = value
elif key == 'servers':
self.servers = value
elif key == 'interfaces':
self.interfaces = value
self.trigger_callback(key)
return
msg_id = response.get('id')
result = response.get('result')
error = response.get('error')
if msg_id is not None:
with self.lock:
method, params, callback = self.unanswered_requests.pop(msg_id)
else:
method = response.get('method')
params = response.get('params')
with self.lock:
for k, v in self.subscriptions.items():
if (method, params) in v:
callback = k
break
else:
print_error("received unexpected notification", method,
params)
return
r = {
'method': method,
'params': params,
'result': result,
'id': msg_id,
'error': error
}
callback(r)
def send(self, messages, callback):
"""return the ids of the requests that we sent"""
# detect subscriptions
sub = []
for message in messages:
m, v = message
if m[-10:] == '.subscribe':
sub.append(message)
if sub:
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in sub:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
with self.lock:
requests = []
ids = []
for m in messages:
method, params = m
request = {
'id': self.message_id,
'method': method,
'params': params
}
self.unanswered_requests[
self.message_id] = method, params, callback
ids.append(self.message_id)
requests.append(request)
if self.debug:
print_error("-->", request)
self.message_id += 1
self.pipe.send_all(requests)
return ids
def synchronous_get(self, requests, timeout=100000000):
queue = Queue.Queue()
ids = self.send(requests, queue.put)
id2 = ids[:]
res = {}
while ids:
r = queue.get(True, timeout)
_id = r.get('id')
ids.remove(_id)
if r.get('error'):
return BaseException(r.get('error'))
result = r.get('result')
res[_id] = r.get('result')
out = []
for _id in id2:
out.append(res[_id])
return out
def get_servers(self):
return self.servers
def get_interfaces(self):
return self.interfaces
def get_header(self, height):
return self.synchronous_get([('network.get_header', [height])])[0]
def get_local_height(self):
return self.blockchain_height
def get_server_height(self):
return self.server_height
def is_connected(self):
return self.status == 'connected'
def is_connecting(self):
return self.status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def get_parameters(self):
return self.synchronous_get([('network.get_parameters', [])])[0]
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server_str = serialize_server(host, port, protocol)
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server_str, True)
# abort if changes were not allowed by config
if self.config.get('server') != server_str or self.config.get(
'proxy') != proxy_str:
return
return self.synchronous_get([('network.set_parameters',
(host, port, protocol, proxy,
auto_connect))])[0]
def stop_daemon(self):
return self.send([('daemon.stop', [])], None)
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event):
with self.lock:
callbacks = self.callbacks.get(event, [])[:]
if callbacks:
[callback() for callback in callbacks]
class Network(threading.Thread):
def __init__(self, config = {}):
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.callbacks = {}
self.protocol = self.config.get('protocol','s')
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = [] # returned by interface (list from irc)
self.disconnected_servers = []
self.recent_servers = self.config.get('recent_servers',[]) # successful connections
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.server_lag = 0
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# default subscriptions
self.subscriptions = {}
self.subscriptions[self.on_banner] = [('server.banner',[])]
self.subscriptions[self.on_peers] = [('server.peers.subscribe',[])]
def is_connected(self):
return self.interface and self.interface.is_connected
def send_subscriptions(self):
for cb, sub in self.subscriptions.items():
self.interface.send(sub, cb)
def subscribe(self, messages, callback):
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in messages:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
if self.interface and self.interface.is_connected:
self.interface.send( messages, callback )
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event):
with self.lock:
callbacks = self.callbacks.get(event,[])[:]
if callbacks:
[callback() for callback in callbacks]
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list:
if not self.interfaces:
# we are probably offline, retry later
self.disconnected_servers = []
return
server = random.choice( choice_list )
return server
def get_servers(self):
out = self.irc_servers if self.irc_servers else DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.interfaces[server] = i
i.start(self.queue)
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
self.interface = self.interfaces[self.default_server]
for i in range(NUM_SERVERS):
self.start_random_interface()
if not self.interface:
self.interface = self.interfaces.values()[0]
def start(self, wait=False):
self.start_interfaces()
threading.Thread.start(self)
if wait:
self.interface.connect_event.wait()
return self.interface.is_connected
def wait_until_connected(self):
while not self.interface:
time.sleep(1)
self.interface.connect_event.wait()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([ host, port, protocol ])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values(): i.stop()
if auto_connect:
self.interface = None
return
if auto_connect:
if not self.interface:
self.switch_to_random_interface()
else:
if self.server_lag > 0:
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
if self.interfaces:
self.switch_to_interface(random.choice(self.interfaces.values()))
def switch_to_interface(self, interface):
assert self.interface is None
server = interface.server
print_error("switching to", server)
self.interface = interface
h = self.heights.get(server)
if h:
self.server_lag = self.blockchain.height() - h
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.trigger_callback('connected')
def stop_interface(self):
self.interface.stop()
self.interface = None
def set_server(self, server):
if self.default_server == server and self.interface:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface:
self.stop_interface()
# notify gui
self.trigger_callback('disconnecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface( self.interfaces[server] )
else:
self.start_interface(server)
self.interface = self.interfaces[server]
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def new_blockchain_height(self, blockchain_height, i):
print_error('new_blockchain_height')
if self.is_connected():
h = self.heights.get(self.interface.server)
if h:
self.server_lag = blockchain_height - h
if self.server_lag > 1:
print_error( "Server is lagging", blockchain_height, h)
if self.config.get('auto_cycle'):
self.set_server(i.server)
else:
print_error('no height for main interface')
self.trigger_callback('updated')
def run(self):
self.blockchain.start()
with self.lock:
self.running = True
while self.is_running():
try:
i = self.queue.get(timeout = 30 if self.interfaces else 3)
except Queue.Empty:
if len(self.interfaces) < NUM_SERVERS:
self.start_random_interface()
continue
if i.is_connected:
self.add_recent_server(i)
i.send([ ('blockchain.headers.subscribe',[])], self.on_header)
if i == self.interface:
print_error('sending subscriptions to', self.interface.server)
self.send_subscriptions()
self.trigger_callback('connected')
else:
self.disconnected_servers.append(i.server)
self.interfaces.pop(i.server)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.interface = None
self.trigger_callback('disconnected')
if self.interface is None and self.config.get('auto_cycle'):
self.switch_to_random_interface()
def on_header(self, i, r):
result = r.get('result')
if not result: return
height = result.get('block_height')
self.heights[i.server] = height
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
self.server_lag = self.blockchain.height() - height
if self.server_lag > 1 and self.config.get('auto_cycle'):
print_error( "Server lagging, stopping interface")
self.stop_interface()
self.trigger_callback('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = self.parse_servers(r.get('result'))
self.trigger_callback('peers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.trigger_callback('banner')
def stop(self):
with self.lock: self.running = False
def is_running(self):
with self.lock: return self.running
def synchronous_get(self, requests, timeout=100000000):
queue = Queue.Queue()
ids = self.interface.send(requests, lambda i,r: queue.put(r))
id2 = ids[:]
res = {}
while ids:
r = queue.get(True, timeout)
_id = r.get('id')
if _id in ids:
ids.remove(_id)
res[_id] = r.get('result')
out = []
for _id in id2:
out.append(res[_id])
return out
def retrieve_transaction(self, tx_hash, tx_height=0):
import transaction
r = self.synchronous_get([ ('blockchain.transaction.get',[tx_hash, tx_height]) ])[0]
if r:
return transaction.Transaction(r)
def parse_servers(self, result):
""" parse servers list into dict format"""
from version import PROTOCOL_VERSION
servers = {}
for item in result:
host = item[1]
out = {}
version = None
pruning_level = '-'
if len(item) > 2:
for v in item[2]:
if re.match("[stgh]\d*", v):
protocol, port = v[0], v[1:]
if port == '': port = DEFAULT_PORTS[protocol]
out[protocol] = port
elif re.match("v(.?)+", v):
version = v[1:]
elif re.match("p\d*", v):
pruning_level = v[1:]
if pruning_level == '': pruning_level = '0'
try:
is_recent = float(version)>=float(PROTOCOL_VERSION)
except:
is_recent = False
if out and is_recent:
out['pruning'] = pruning_level
servers[host] = out
return servers
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote electrum
servers, each connected socket is handled by an Interface() object.
Connections are initiated by a Connection() thread which stops once
the connection succeeds or fails.
Our external API:
- Member functions get_header(), get_interfaces(), get_local_height(),
get_parameters(), get_server_height(), get_status_value(),
is_connected(), set_parameters(), stop()
"""
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 10 if not self.config.get('oneserver') else 0
self.blockchains = blockchain.read_blockchains(self.config)
self.print_error("blockchains", self.blockchains.keys())
self.blockchain_index = config.get('blockchain_index', 0)
if self.blockchain_index not in self.blockchains.keys():
self.blockchain_index = 0
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.lock = threading.Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.donation_address = ''
self.relay_fee = None
# List of all proposals on the network.
self.all_proposals = []
# callbacks passed with subscriptions
self.subscriptions = defaultdict(list)
self.sub_cache = {}
# callbacks set by the GUI
self.callbacks = defaultdict(list)
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# Servers that have invalid versions.
self.invalid_version_servers = set()
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', True)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def register_callback(self, callback, events):
with self.lock:
for event in events:
self.callbacks[event].append(callback)
def unregister_callback(self, callback):
with self.lock:
for callbacks in self.callbacks.values():
if callback in callbacks:
callbacks.remove(callback)
def trigger_callback(self, event, *args):
with self.lock:
callbacks = self.callbacks[event][:]
[callback(event, *args) for callback in callbacks]
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.interface.tip if self.interface else 0
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return True
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface is not None
def is_connecting(self):
return self.connection_status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def queue_request(self, method, params, interface=None):
# If you want to queue a request on any interface it must go
# through this function so message ids are properly tracked
if interface is None:
interface = self.interface
message_id = self.message_id
self.message_id += 1
if self.debug:
self.print_error(interface.host, "-->", method, params, message_id)
interface.queue_request(method, params, message_id)
return message_id
def send_subscriptions(self):
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
self.sub_cache.clear()
# Resend unanswered requests
requests = self.unanswered_requests.values()
self.unanswered_requests = {}
for request in requests:
message_id = self.queue_request(request[0], request[1])
self.unanswered_requests[message_id] = request
self.queue_request('server.banner', [])
self.queue_request('server.donation_address', [])
self.queue_request('server.peers.subscribe', [])
for i in bitcoin.FEE_TARGETS:
self.queue_request('blockchain.estimatefee', [i])
self.queue_request('blockchain.relayfee', [])
for addr in self.subscribed_addresses:
self.queue_request('blockchain.address.subscribe', [addr])
# Disabled until API is stable.
# self.queue_request('masternode.proposals.subscribe', [])
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'fee':
value = self.config.fee_estimates
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
if key in ['status', 'updated']:
self.trigger_callback(key)
else:
self.trigger_callback(key, self.get_status_value(key))
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect
def get_donation_address(self):
if self.is_connected():
return self.donation_address
def get_interfaces(self):
'''The interfaces that are in connected state'''
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers.copy()
out.update(DEFAULT_SERVERS)
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if (not server in self.interfaces and not server in self.connecting):
if server == self.default_server:
self.print_error("connecting to %s as new interface" % server)
self.set_status('connecting')
self.connecting.add(server)
c = Connection(server, self.socket_queue, self.config.path)
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
exclude_set = self.invalid_version_servers.union(exclude_set)
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
self.print_error('setting proxy', proxy)
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode,
proxy["host"],
int(proxy["port"]),
# socks.py seems to want either None or a non-empty string
username=(proxy.get("user", "") or None),
password=(proxy.get("password", "") or None))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
assert not self.connecting and self.socket_queue.empty()
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for interface in self.interfaces.values():
self.close_interface(interface)
if self.interface:
self.close_interface(self.interface)
assert self.interface is None
assert not self.interfaces
self.connecting = set()
# Get a new queue - no old pending connections thanks!
self.socket_queue = Queue.Queue()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server = serialize_server(host, port, protocol)
# sanitize parameters
try:
deserialize_server(serialize_server(host, port, protocol))
if proxy:
proxy_modes.index(proxy["mode"]) + 1
int(proxy['port'])
except:
return
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server, True)
# abort if changes were not allowed by config
if self.config.get('server') != server or self.config.get('proxy') != proxy_str:
return
self.auto_connect = auto_connect
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
self.notify('updated')
def switch_to_random_interface(self):
'''Switch to a random connected server other than the current one'''
servers = self.get_interfaces() # Those in connected state
if self.default_server in servers:
servers.remove(self.default_server)
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
# switch to one that has the correct header (not height)
header = self.blockchain().read_header(self.get_local_height())
filtered = map(lambda x:x[0], filter(lambda x: x[1].tip_header==header, self.interfaces.items()))
if filtered:
choice = random.choice(filtered)
self.switch_to_interface(choice)
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.interface = None
self.start_interface(server)
return
i = self.interfaces[server]
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
# fixme: we don't want to close headers sub
#self.close_interface(self.interface)
self.interface = i
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def close_interface(self, interface):
if interface:
if interface.server in self.interfaces:
self.interfaces.pop(interface.server)
if interface.server == self.default_server:
self.interface = None
interface.close()
def add_recent_server(self, server):
# list is ordered
if server in self.recent_servers:
self.recent_servers.remove(server)
self.recent_servers.insert(0, server)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def process_response(self, interface, response, callbacks):
if self.debug:
self.print_error("<--", response)
error = response.get('error')
result = response.get('result')
method = response.get('method')
params = response.get('params')
# We handle some responses; return the rest to the client.
if method == 'server.version':
if error is None:
self.on_version(interface, result)
elif method == 'blockchain.headers.subscribe':
if error is None:
self.on_notify_header(interface, result)
elif method == 'server.peers.subscribe':
if error is None:
self.irc_servers = parse_servers(result)
self.notify('servers')
# elif method == 'masternode.proposals.subscribe':
# if error is None:
# self.on_proposals(result)
elif method == 'server.banner':
if error is None:
self.banner = result
self.notify('banner')
elif method == 'server.donation_address':
if error is None:
self.donation_address = result
elif method == 'blockchain.estimatefee':
if error is None and result > 0:
i = params[0]
fee = int(result*COIN)
self.config.fee_estimates[i] = fee
self.print_error("fee_estimates[%d]" % i, fee)
self.notify('fee')
elif method == 'blockchain.relayfee':
if error is None:
self.relay_fee = int(result * COIN)
self.print_error("relayfee", self.relay_fee)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(interface, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(interface, response)
for callback in callbacks:
callback(response)
def get_index(self, method, params):
""" hashable index for subscriptions and cache"""
return str(method) + (':' + str(params[0]) if params else '')
def process_responses(self, interface):
responses = interface.get_responses()
for request, response in responses:
if request:
method, params, message_id = request
k = self.get_index(method, params)
# client requests go through self.send() with a
# callback, are only sent to the current interface,
# and are placed in the unanswered_requests dictionary
client_req = self.unanswered_requests.pop(message_id, None)
if client_req:
assert interface == self.interface
callbacks = [client_req[2]]
else:
# fixme: will only work for subscriptions
k = self.get_index(method, params)
callbacks = self.subscriptions.get(k, [])
# Copy the request method and params to the response
response['method'] = method
response['params'] = params
# Only once we've received a response to an addr subscription
# add it to the list; avoids double-sends on reconnection
if method == 'blockchain.address.subscribe':
self.subscribed_addresses.add(params[0])
else:
if not response: # Closed remotely / misbehaving
self.connection_down(interface.server)
break
# Rewrite response shape to match subscription request response
method = response.get('method')
params = response.get('params')
k = self.get_index(method, params)
if method == 'blockchain.headers.subscribe':
response['result'] = params[0]
response['params'] = []
elif method == 'blockchain.address.subscribe':
response['params'] = [params[0]] # addr
response['result'] = params[1]
callbacks = self.subscriptions.get(k, [])
# update cache if it's a subscription
if method.endswith('.subscribe'):
self.sub_cache[k] = response
# Response is now in canonical form
self.process_response(interface, response, callbacks)
def send(self, messages, callback):
'''Messages is a list of (method, params) tuples'''
with self.lock:
self.pending_sends.append((messages, callback))
def process_pending_sends(self):
# Requests needs connectivity. If we don't have an interface,
# we cannot process them.
if not self.interface:
return
with self.lock:
sends = self.pending_sends
self.pending_sends = []
for messages, callback in sends:
for method, params in messages:
r = None
if method.endswith('.subscribe'):
k = self.get_index(method, params)
# add callback to list
l = self.subscriptions.get(k, [])
if callback not in l:
l.append(callback)
self.subscriptions[k] = l
# check cached response for subscriptions
r = self.sub_cache.get(k)
if r is not None:
util.print_error("cache hit", k)
callback(r)
else:
message_id = self.queue_request(method, params)
self.unanswered_requests[message_id] = method, params, callback
def unsubscribe(self, callback):
'''Unsubscribe a callback to free object references to enable GC.'''
# Note: we can't unsubscribe from the server, so if we receive
# subsequent notifications process_response() will emit a harmless
# "received unexpected notification" warning
with self.lock:
for v in self.subscriptions.values():
if callback in v:
v.remove(callback)
def invalid_version(self, server):
'''A server has an incompatible version.'''
self.invalid_version_servers.add(server)
self.connection_down(server)
def connection_down(self, server):
'''A connection to server either went down, or was never made.
We distinguish by whether it is in self.interfaces.'''
self.disconnected_servers.add(server)
if server == self.default_server:
self.set_status('disconnected')
if server in self.interfaces:
self.close_interface(self.interfaces[server])
self.notify('interfaces')
for b in self.blockchains.values():
if b.catch_up == server:
b.catch_up = None
def new_interface(self, server, socket):
# todo: get tip first, then decide which checkpoint to use.
self.add_recent_server(server)
interface = Interface(server, socket)
interface.blockchain = None
interface.tip_header = None
interface.tip = 0
interface.mode = 'default'
interface.request = None
self.interfaces[server] = interface
self.queue_request('server.version',
[ELECTRUM_VERSION, PROTOCOL_VERSION], interface)
self.queue_request('blockchain.headers.subscribe', [], interface)
if server == self.default_server:
self.switch_to_interface(server)
#self.notify('interfaces')
def maintain_sockets(self):
'''Socket maintenance.'''
# Responses to connection attempts?
while not self.socket_queue.empty():
server, socket = self.socket_queue.get()
if server in self.connecting:
self.connecting.remove(server)
if socket:
self.new_interface(server, socket)
else:
self.connection_down(server)
# Send pings and shut down stale interfaces
for interface in self.interfaces.values():
if interface.has_timed_out():
self.connection_down(interface.server)
elif interface.ping_required():
params = [ELECTRUM_VERSION, PROTOCOL_VERSION]
self.queue_request('server.version', params, interface)
now = time.time()
# nodes
if len(self.interfaces) + len(self.connecting) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
if not self.is_connecting():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, idx):
interface.print_error("requesting chunk %d" % idx)
self.queue_request('blockchain.block.get_chunk', [idx], interface)
interface.request = idx
interface.req_time = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
error = response.get('error')
result = response.get('result')
params = response.get('params')
if result is None or params is None or error is not None:
interface.print_error(error or 'bad response')
return
# Ignore unsolicited chunks
index = params[0]
if interface.request != index:
return
connect = interface.blockchain.connect_chunk(index, result)
# If not finished, get the next chunk
if not connect:
self.connection_down(interface.server)
return
if interface.blockchain.height() < interface.tip:
self.request_chunk(interface, index+1)
else:
interface.request = None
interface.mode = 'default'
interface.print_error('catch up done', interface.blockchain.height())
interface.blockchain.catch_up = None
self.notify('updated')
def request_header(self, interface, height):
#interface.print_error("requesting header %d" % height)
self.queue_request('blockchain.block.get_header', [height], interface)
interface.request = height
interface.req_time = time.time()
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
header = response.get('result')
if not header:
interface.print_error(response)
self.connection_down(interface.server)
return
height = header.get('block_height')
if interface.request != height:
interface.print_error("unsolicited header",interface.request, height)
self.connection_down(interface.server)
return
chain = blockchain.check_header(header)
if interface.mode == 'backward':
if chain:
interface.print_error("binary search")
interface.mode = 'binary'
interface.blockchain = chain
interface.good = height
next_height = (interface.bad + interface.good) // 2
else:
if height == 0:
self.connection_down(interface.server)
next_height = None
else:
interface.bad = height
interface.bad_header = header
delta = interface.tip - height
next_height = max(0, interface.tip - 2 * delta)
elif interface.mode == 'binary':
if chain:
interface.good = height
interface.blockchain = chain
else:
interface.bad = height
interface.bad_header = header
if interface.bad != interface.good + 1:
next_height = (interface.bad + interface.good) // 2
elif not interface.blockchain.can_connect(interface.bad_header, check_height=False):
self.connection_down(interface.server)
next_height = None
else:
branch = self.blockchains.get(interface.bad)
if branch is not None:
if branch.check_header(interface.bad_header):
interface.print_error('joining chain', interface.bad)
next_height = None
elif branch.parent().check_header(header):
interface.print_error('reorg', interface.bad, interface.tip)
interface.blockchain = branch.parent()
next_height = None
else:
interface.print_error('checkpoint conflicts with existing fork', branch.path())
branch.write('', 0)
branch.save_header(interface.bad_header)
interface.mode = 'catch_up'
interface.blockchain = branch
next_height = interface.bad + 1
interface.blockchain.catch_up = interface.server
else:
bh = interface.blockchain.height()
next_height = None
if bh > interface.good:
if not interface.blockchain.check_header(interface.bad_header):
b = interface.blockchain.fork(interface.bad_header)
self.blockchains[interface.bad] = b
interface.blockchain = b
interface.print_error("new chain", b.checkpoint)
interface.mode = 'catch_up'
next_height = interface.bad + 1
interface.blockchain.catch_up = interface.server
else:
assert bh == interface.good
if interface.blockchain.catch_up is None and bh < interface.tip:
interface.print_error("catching up from %d"% (bh + 1))
interface.mode = 'catch_up'
next_height = bh + 1
interface.blockchain.catch_up = interface.server
self.notify('updated')
elif interface.mode == 'catch_up':
can_connect = interface.blockchain.can_connect(header)
if can_connect:
interface.blockchain.save_header(header)
next_height = height + 1 if height < interface.tip else None
else:
# go back
interface.print_error("cannot connect", height)
interface.mode = 'backward'
interface.bad = height
interface.bad_header = header
next_height = height - 1
if next_height is None:
# exit catch_up state
interface.print_error('catch up done', interface.blockchain.height())
interface.blockchain.catch_up = None
self.switch_lagging_interface()
self.notify('updated')
elif interface.mode == 'default':
if not ok:
interface.print_error("default: cannot connect %d"% height)
interface.mode = 'backward'
interface.bad = height
interface.bad_header = header
next_height = height - 1
else:
interface.print_error("we are ok", height, interface.request)
next_height = None
else:
raise BaseException(interface.mode)
# If not finished, get the next header
if next_height:
if interface.mode == 'catch_up' and interface.tip > next_height + 50:
self.request_chunk(interface, next_height // 2016)
else:
self.request_header(interface, next_height)
else:
interface.mode = 'default'
interface.request = None
self.notify('updated')
# refresh network dialog
self.notify('interfaces')
def maintain_requests(self):
for interface in self.interfaces.values():
if interface.request and time.time() - interface.request_time > 20:
interface.print_error("blockchain request timed out")
self.connection_down(interface.server)
continue
def wait_on_sockets(self):
# Python docs say Windows doesn't like empty selects.
# Sleep to prevent busy looping
if not self.interfaces:
time.sleep(0.1)
return
rin = [i for i in self.interfaces.values()]
win = [i for i in self.interfaces.values() if i.num_requests()]
try:
rout, wout, xout = select.select(rin, win, [], 0.1)
except socket.error as (code, msg):
if code == errno.EINTR:
return
raise
assert not xout
for interface in wout:
interface.send_requests()
for interface in rout:
self.process_responses(interface)
class Network(threading.Thread):
def __init__(self, config = {}):
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.callbacks = {}
self.protocol = self.config.get('protocol','s')
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = [] # returned by interface (list from irc)
self.pending_servers = set([])
self.disconnected_servers = set([])
self.recent_servers = self.config.get('recent_servers',[]) # successful connections
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.server_lag = 0
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# default subscriptions
self.subscriptions = {}
self.subscriptions[self.on_banner] = [('server.banner',[])]
self.subscriptions[self.on_peers] = [('server.peers.subscribe',[])]
self.pending_transactions_for_notifications = []
def is_connected(self):
return self.interface and self.interface.is_connected
def is_up_to_date(self):
return self.interface.is_up_to_date()
def main_server(self):
return self.interface.server
def send_subscriptions(self):
for cb, sub in self.subscriptions.items():
self.interface.send(sub, cb)
def subscribe(self, messages, callback):
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in messages:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
if self.is_connected():
self.interface.send( messages, callback )
def send(self, messages, callback):
if self.is_connected():
self.interface.send( messages, callback )
return True
else:
return False
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event):
with self.lock:
callbacks = self.callbacks.get(event,[])[:]
if callbacks:
[callback() for callback in callbacks]
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list:
if not self.interfaces:
# we are probably offline, retry later
self.disconnected_servers = set([])
return
server = random.choice( choice_list )
return server
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.pending_servers.add(server)
i.start(self.queue)
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, wait=False):
self.start_interfaces()
threading.Thread.start(self)
if wait:
return self.wait_until_connected()
def wait_until_connected(self):
"wait until connection status is known"
if self.config.get('auto_cycle'):
# self.random_server() returns None if all servers have been tried
while not self.is_connected() and self.random_server():
time.sleep(0.1)
else:
self.interface.connect_event.wait()
return self.interface.is_connected
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([ host, port, protocol ])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values(): i.stop()
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected:
self.switch_to_random_interface()
else:
if self.server_lag > 0:
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
if self.interfaces:
self.switch_to_interface(random.choice(self.interfaces.values()))
def switch_to_interface(self, interface):
assert not self.interface.is_connected
server = interface.server
print_error("switching to", server)
self.interface = interface
h = self.heights.get(server)
if h:
self.server_lag = self.blockchain.height() - h
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.trigger_callback('connected')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected:
self.stop_interface()
# notify gui
self.trigger_callback('disconnecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface( self.interfaces[server] )
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
h = self.heights.get(self.interface.server)
if h:
self.server_lag = blockchain_height - h
if self.server_lag > 1:
print_error( "Server is lagging", blockchain_height, h)
if self.config.get('auto_cycle'):
self.set_server(i.server)
else:
print_error('no height for main interface')
self.trigger_callback('updated')
def run(self):
self.blockchain.start()
with self.lock:
self.running = True
while self.is_running():
try:
i = self.queue.get(timeout = 30 if self.interfaces else 3)
except Queue.Empty:
if len(self.interfaces) < self.num_server:
self.start_random_interface()
continue
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected:
#if i.server in self.interfaces: raise
self.interfaces[i.server] = i
self.add_recent_server(i)
i.send([ ('blockchain.headers.subscribe',[])], self.on_header)
if i == self.interface:
print_error('sending subscriptions to', self.interface.server)
self.send_subscriptions()
self.trigger_callback('connected')
else:
self.disconnected_servers.add(i.server)
if i.server in self.interfaces:
self.interfaces.pop(i.server)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
#self.interface = None
self.trigger_callback('disconnected')
if not self.interface.is_connected and self.config.get('auto_cycle'):
self.switch_to_random_interface()
def on_header(self, i, r):
result = r.get('result')
if not result: return
height = result.get('block_height')
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
self.server_lag = self.blockchain.height() - height
if self.server_lag > 1 and self.config.get('auto_cycle'):
print_error( "Server lagging, stopping interface")
self.stop_interface()
self.trigger_callback('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.trigger_callback('peers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.trigger_callback('banner')
def stop(self):
with self.lock: self.running = False
def is_running(self):
with self.lock: return self.running
def synchronous_get(self, requests, timeout=100000000):
return self.interface.synchronous_get(requests)
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote lbryum
servers, each connected socket is handled by an Interface() object.
Connections are initiated by a Connection() thread which stops once
the connection succeeds or fails.
Our external API:
- Member functions get_header(), get_interfaces(), get_local_height(),
get_parameters(), get_server_height(), get_status_value(),
is_connected(), set_parameters(), stop()
"""
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = get_blockchain(self.config, self)
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
default_servers = self.config.get('default_servers')
if not default_servers:
raise ValueError('No servers have been specified')
self.default_server = pick_random_server(default_servers)
self.lock = Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.banner = ''
self.fee = None
self.relay_fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
# catchup counter, used to track catchup progress before chain is verified and headers saved
self.catchup_progress = 0
# callbacks passed with subscriptions
self.subscriptions = defaultdict(list)
self.sub_cache = {}
# callbacks set by the GUI
self.callbacks = defaultdict(list)
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.online_servers = {}
self._set_online_servers()
self.start_network(
deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def register_callback(self, callback, events):
with self.lock:
for event in events:
self.callbacks[event].append(callback)
def unregister_callback(self, callback):
with self.lock:
for callbacks in self.callbacks.values():
if callback in callbacks:
callbacks.remove(callback)
def trigger_callback(self, event, *args):
with self.lock:
callbacks = self.callbacks[event][:]
[callback(event, *args) for callback in callbacks]
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
log.info('no height for main interface')
return True
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
log.info('%s is lagging (%d vs %d)', self.default_server, sh, lh)
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface is not None
def is_connecting(self):
return self.connection_status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def queue_request(self, method, params, interface=None):
# If you want to queue a request on any interface it must go
# through this function so message ids are properly tracked
if interface is None:
interface = self.interface
message_id = self.message_id
self.message_id += 1
if self.debug:
log.debug('%s --> %s, %s, %s', interface.host, method, params,
message_id)
interface.queue_request(method, params, message_id)
return message_id
def send_subscriptions(self):
log.info(
'sending subscriptions to %s. Unanswered requests: %s, Subscribed addresses: %s',
self.interface.server, len(self.unanswered_requests),
len(self.subscribed_addresses))
self.sub_cache.clear()
# Resend unanswered requests
requests = self.unanswered_requests.values()
self.unanswered_requests = {}
for request in requests:
message_id = self.queue_request(request[0], request[1])
self.unanswered_requests[message_id] = request
for addr in self.subscribed_addresses:
self.queue_request('blockchain.address.subscribe', [addr])
self.queue_request('server.banner', [])
self.queue_request('server.peers.subscribe', [])
self.queue_request('blockchain.estimatefee', [2])
self.queue_request('blockchain.relayfee', [])
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'fee':
value = self.fee
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
if key in ['status', 'updated']:
self.trigger_callback(key)
else:
self.trigger_callback(key, self.get_status_value(key))
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect
def get_interfaces(self):
'''The interfaces that are in connected state'''
return self.interfaces.keys()
#Do an initial pruning of lbryum servers that don't have the specified port open
def _set_online_servers(self):
servers = self.config.get('default_servers', {}).iteritems()
self.online_servers = {
host: ports
for host, ports in servers if is_online(host, ports)
}
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = self.online_servers
return out
def start_interface(self, server):
if not server in self.interfaces and not server in self.connecting:
if server == self.default_server:
log.info("connecting to %s as new interface", server)
self.set_status('connecting')
self.connecting.add(server)
c = Connection(server, self.socket_queue, self.config.path)
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol,
exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
log.info('setting proxy %s', proxy)
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"],
int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(
socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
assert not self.connecting and self.socket_queue.empty()
log.info('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
log.info("stopping network")
for interface in self.interfaces.values():
self.close_interface(interface)
assert self.interface is None
assert not self.interfaces
self.connecting = set()
# Get a new queue - no old pending connections thanks!
self.socket_queue = Queue.Queue()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server = serialize_server(host, port, protocol)
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server, True)
# abort if changes were not allowed by config
if self.config.get('server') != server or self.config.get(
'proxy') != proxy_str:
return
self.auto_connect = auto_connect
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
'''Switch to a random connected server other than the current one'''
servers = self.get_interfaces() # Those in connected state
if self.default_server in servers:
servers.remove(self.default_server)
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion=None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
if suggestion and self.protocol == deserialize_server(
suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.interface = None
self.start_interface(server)
return
i = self.interfaces[server]
if self.interface != i:
log.info("switching to %s", server)
# stop any current interface in order to terminate subscriptions
self.close_interface(self.interface)
self.interface = i
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def close_interface(self, interface):
if interface:
self.interfaces.pop(interface.server)
if interface.server == self.default_server:
self.interface = None
interface.close()
def process_response(self, interface, response, callbacks):
if self.debug:
log.debug("<-- %s", response)
error = response.get('error')
result = response.get('result')
method = response.get('method')
params = response.get('params')
# We handle some responses; return the rest to the client.
if method == 'server.version':
interface.server_version = result
elif method == 'blockchain.headers.subscribe':
if error is None:
self.on_header(interface, result)
elif method == 'server.peers.subscribe':
if error is None:
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
if error is None:
self.banner = result
self.notify('banner')
elif method == 'blockchain.estimatefee':
if error is None:
self.fee = int(result * COIN)
log.info("recommended fee %s", self.fee)
self.notify('fee')
elif method == 'blockchain.relayfee':
if error is None:
self.relay_fee = int(result * COIN)
log.info("relayfee %s", self.relay_fee)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(interface, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(interface, response)
for callback in callbacks:
callback(response)
def get_index(self, method, params):
""" hashable index for subscriptions and cache"""
return str(method) + (':' + str(params[0]) if params else '')
def process_responses(self, interface):
responses = interface.get_responses()
for request, response in responses:
if request:
method, params, message_id = request
k = self.get_index(method, params)
# client requests go through self.send() with a
# callback, are only sent to the current interface,
# and are placed in the unanswered_requests dictionary
client_req = self.unanswered_requests.pop(message_id, None)
if client_req:
assert interface == self.interface
callbacks = [client_req[2]]
else:
callbacks = []
# Copy the request method and params to the response
response['method'] = method
response['params'] = params
# Only once we've received a response to an addr subscription
# add it to the list; avoids double-sends on reconnection
if method == 'blockchain.address.subscribe':
self.subscribed_addresses.add(params[0])
else:
if not response: # Closed remotely / misbehaving
self.connection_down(interface.server)
break
# Rewrite response shape to match subscription request response
method = response.get('method')
params = response.get('params')
k = self.get_index(method, params)
if method == 'blockchain.headers.subscribe':
response['result'] = params[0]
response['params'] = []
elif method == 'blockchain.address.subscribe':
response['params'] = [params[0]] # addr
response['result'] = params[1]
callbacks = self.subscriptions.get(k, [])
# update cache if it's a subscription
if method.endswith('.subscribe'):
self.sub_cache[k] = response
# Response is now in canonical form
self.process_response(interface, response, callbacks)
def send(self, messages, callback):
'''Messages is a list of (method, params) tuples'''
with self.lock:
self.pending_sends.append((messages, callback))
def process_pending_sends(self):
# Requests needs connectivity. If we don't have an interface,
# we cannot process them.
if not self.interface:
return
with self.lock:
sends = self.pending_sends
self.pending_sends = []
for messages, callback in sends:
for method, params in messages:
r = None
if method.endswith('.subscribe'):
k = self.get_index(method, params)
# add callback to list
l = self.subscriptions.get(k, [])
if callback not in l:
l.append(callback)
self.subscriptions[k] = l
# check cached response for subscriptions
r = self.sub_cache.get(k)
if r is not None:
util.print_error("cache hit", k)
callback(r)
else:
message_id = self.queue_request(method, params)
self.unanswered_requests[
message_id] = method, params, callback
def unsubscribe(self, callback):
'''Unsubscribe a callback to free object references to enable GC.'''
# Note: we can't unsubscribe from the server, so if we receive
# subsequent notifications process_response() will emit a harmless
# "received unexpected notification" warning
with self.lock:
for v in self.subscriptions.values():
if callback in v:
v.remove(callback)
def connection_down(self, server):
'''A connection to server either went down, or was never made.
We distinguish by whether it is in self.interfaces.'''
self.disconnected_servers.add(server)
if server == self.default_server:
self.set_status('disconnected')
if server in self.interfaces:
self.close_interface(self.interfaces[server])
self.heights.pop(server, None)
self.notify('interfaces')
def new_interface(self, server, socket):
self.interfaces[server] = interface = Interface(server, socket)
self.queue_request('blockchain.headers.subscribe', [], interface)
if server == self.default_server:
self.switch_to_interface(server)
self.notify('interfaces')
def maintain_sockets(self):
'''Socket maintenance.'''
# Responses to connection attempts?
while not self.socket_queue.empty():
server, socket = self.socket_queue.get()
self.connecting.remove(server)
if socket:
self.new_interface(server, socket)
else:
self.connection_down(server)
# Send pings and shut down stale interfaces
for interface in self.interfaces.values():
if interface.has_timed_out():
self.connection_down(interface.server)
elif interface.ping_required():
params = [LBRYUM_VERSION, PROTOCOL_VERSION]
self.queue_request('server.version', params, interface)
now = time.time()
# nodes
if len(self.interfaces) + len(self.connecting) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
log.info('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
if not self.is_connecting():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
log.debug("requesting chunk %d" % idx)
self.queue_request('blockchain.block.get_chunk', [idx], interface)
data['chunk_idx'] = idx
data['req_time'] = time.time()
def _caught_up_to_interface(self, data):
return self.get_local_height() >= data['if_height']
def _need_chunk_from_interface(self, data):
return self.get_local_height() + BLOCKS_PER_CHUNK <= data['if_height']
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx,
response['result'])
if idx < 0 or self._caught_up_to_interface(data):
self.bc_requests.popleft()
self.notify('updated')
elif self._need_chunk_from_interface(data):
self.request_chunk(interface, data, idx)
else:
self.request_header(interface, data, data['if_height'])
def request_header(self, interface, data, height):
log.debug("requesting header %d" % height)
self.queue_request('blockchain.block.get_header', [height], interface)
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(
data['chain'], response['result'])
self.catchup_progress += 1
# If not finished, get the next header
if next_height is True or next_height is False:
self.catchup_progress = 0
self.bc_requests.popleft()
if next_height:
self.switch_lagging_interface(interface.server)
self.notify('updated')
else:
interface.print_error(
"header didn't connect, dismissing interface")
interface.close()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them,
if necessary.
'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height < local_height:
return False
elif if_height > local_height + BLOCKS_PER_CHUNK:
self.request_chunk(interface, data,
(local_height + 1) / BLOCKS_PER_CHUNK)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object.
'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface in self.interfaces.values():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 30:
interface.print_error("blockchain request timed out")
self.connection_down(interface.server)
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def wait_on_sockets(self):
# Python docs say Windows doesn't like empty selects.
# Sleep to prevent busy looping
if not self.interfaces:
time.sleep(0.1)
return
rin = [i for i in self.interfaces.values()]
win = [i for i in self.interfaces.values() if i.unsent_requests]
try:
rout, wout, xout = select.select(rin, win, [], 0.2)
except socket.error as (code, msg):
if code == errno.EINTR:
return
raise
assert not xout
for interface in wout:
interface.send_requests()
for interface in rout:
self.process_responses(interface)
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote
electrum servers, each connection is handled by its own
thread object returned from Interface(). Its external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 8 if not self.config.get("oneserver") else 0
self.blockchain = Blockchain(self.config, self)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get("server")
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ""
self.fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join(self.config.path, "certs")
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get("auto_connect", False)
self.start_network(deserialize_server(self.default_server)[2], deserialize_proxy(self.config.get("proxy")))
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error("no height for main interface")
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error("%s is lagging (%d vs %d)" % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify("status")
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error(
"sending subscriptions to",
self.interface.server,
len(self.unanswered_requests),
len(self.subscribed_addresses),
)
for r in self.unanswered_requests.values():
self.interface.send_request(r)
for addr in self.subscribed_addresses:
self.interface.send_request({"method": "blockchain.address.subscribe", "params": [addr]})
self.interface.send_request({"method": "server.banner", "params": []})
self.interface.send_request({"method": "server.peers.subscribe", "params": []})
self.interface.send_request({"method": "blockchain.estimatefee", "params": [2]})
def get_status_value(self, key):
if key == "status":
value = self.connection_status
elif key == "banner":
value = self.banner
elif key == "fee":
value = self.fee
elif key == "updated":
value = (self.get_local_height(), self.get_server_height())
elif key == "servers":
value = self.get_servers()
elif key == "interfaces":
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({"method": "network.status", "params": [key, value]})
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect
def get_interfaces(self):
"""The interfaces that are in connected state"""
return [s for s, i in self.interfaces.items() if i.is_connected()]
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if not server in self.interfaces.keys():
if server == self.default_server:
self.set_status("connecting")
i = interface.Interface(server, self.queue, self.config)
self.interfaces[i.server] = i
i.start()
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
self.print_error("starting network")
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for i in self.interfaces.values():
i.stop()
self.interface = None
self.interfaces = {}
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.auto_connect = auto_connect
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
servers = self.get_interfaces() # Those in connected state
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion=None):
"""If auto_connect and lagging, switch interface"""
if self.server_is_lagging() and self.auto_connect:
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
"""Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface."""
self.default_server = server
if server not in self.interfaces:
self.print_error("starting %s; will switch once connected" % server)
self.start_interface(server)
return
i = self.interfaces[server]
if not i.is_connected():
# do nothing; we will switch once connected
return
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.stop_interface()
self.interface = i
self.addr_responses = {}
self.send_subscriptions()
self.set_status("connected")
self.notify("updated")
def stop_interface(self):
if self.interface:
self.interface.stop()
self.interface = None
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0, s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify("updated")
def process_if_notification(self, i):
"""Handle interface addition and removal through notifications"""
if i.is_connected():
self.add_recent_server(i)
i.send_request({"method": "blockchain.headers.subscribe", "params": []})
if i.server == self.default_server:
self.switch_to_interface(i.server)
else:
self.interfaces.pop(i.server, None)
self.heights.pop(i.server, None)
if i == self.interface:
self.interface = None
self.addr_responses = {}
self.set_status("disconnected")
self.disconnected_servers.add(i.server)
# Our set of interfaces changed
self.notify("interfaces")
def process_response(self, i, response):
# the id comes from the daemon or the network proxy
_id = response.get("id")
if _id is not None:
if i != self.interface:
return
self.unanswered_requests.pop(_id)
method = response.get("method")
result = response.get("result")
if method == "blockchain.headers.subscribe":
self.on_header(i, response)
elif method == "server.peers.subscribe":
self.irc_servers = parse_servers(result)
self.notify("servers")
elif method == "server.banner":
self.banner = result
self.notify("banner")
elif method == "blockchain.estimatefee":
from bitcoin import COIN
self.fee = int(result * COIN)
self.print_error("recommended fee", self.fee)
self.notify("fee")
elif method == "blockchain.address.subscribe":
addr = response.get("params")[0]
self.addr_responses[addr] = result
self.response_queue.put(response)
elif method == "blockchain.block.get_chunk":
self.on_get_chunk(i, response)
elif method == "blockchain.block.get_header":
self.on_get_header(i, response)
else:
self.response_queue.put(response)
def handle_requests(self):
"""Some requests require connectivity, others we handle locally in
process_request() and must do so in order to e.g. prevent the
daemon seeming unresponsive.
"""
unhandled = []
while not self.requests_queue.empty():
request = self.requests_queue.get()
if not self.process_request(request):
unhandled.append(request)
for request in unhandled:
self.requests_queue.put(request)
def process_request(self, request):
"""Returns true if the request was processed."""
method = request["method"]
params = request["params"]
_id = request["id"]
if method.startswith("network."):
out = {"id": _id}
try:
f = getattr(self, method[8:])
out["result"] = f(*params)
except AttributeError:
out["error"] = "unknown method"
except BaseException as e:
out["error"] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return True
if method == "blockchain.address.subscribe":
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({"id": _id, "result": self.addr_responses[addr]})
return True
# This request needs connectivity. If we don't have an
# interface, we cannot process it.
if not self.is_connected():
return False
self.unanswered_requests[_id] = request
self.interface.send_request(request)
return True
def check_interfaces(self):
now = time.time()
# nodes
if len(self.interfaces) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error("network: retrying connections")
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
interface.send_request({"method": "blockchain.block.get_chunk", "params": [idx]})
data["chunk_idx"] = idx
data["req_time"] = time.time()
def on_get_chunk(self, interface, response):
"""Handle receiving a chunk of block headers"""
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get("chunk_idx")
# Ignore unsolicited chunks
if req_if == interface and req_idx == response["params"][0]:
idx = self.blockchain.connect_chunk(req_idx, response["result"])
# If not finished, get the next chunk
if idx < 0 or self.get_local_height() >= data["if_height"]:
self.bc_requests.popleft()
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
interface.send_request({"method": "blockchain.block.get_header", "params": [height]})
data["header_height"] = height
data["req_time"] = time.time()
if not "chain" in data:
data["chain"] = []
def on_get_header(self, interface, response):
"""Handle receiving a single block header"""
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get("header_height", -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response["params"][0]:
next_height = self.blockchain.connect_header(data["chain"], response["result"])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify("updated")
else:
interface.print_error("header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
"""Send a request for the next header, or a chunk of them, if necessary"""
local_height, if_height = self.get_local_height(), data["if_height"]
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) / 2016)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
"""Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object"""
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface.is_connected():
continue
req_time = data.get("req_time")
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
interface.stop()
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def run(self):
self.blockchain.init()
while self.is_running():
self.check_interfaces()
self.handle_requests()
self.handle_bc_requests()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
# if response is None it is a notification about the interface
if response is None:
self.process_if_notification(i)
else:
self.process_response(i, response)
self.stop_network()
self.print_error("stopped")
def on_header(self, i, r):
header = r.get("result")
if not header:
return
height = header.get("block_height")
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get("merkle_root")
self.utxo_roots[i.server] = header.get("utxo_root")
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {"if_height": height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify("updated")
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()