def test_idempotency(self):
blockchain = Blockchain()
blockchain.register_node('http://192.168.0.1:5000')
blockchain.register_node('http://192.168.0.1:5000')
assert len(blockchain.nodes) == 1
def test_add_multiple_blocks_sets_hashes_correctly(self):
chain = Blockchain()
chain.blocks = self.get_blocks("first", "second", "third")
self.assertEqual(3, len(chain.blocks))
self.assertEqual("first", chain.blocks[0].messages[0].data)
self.assertEqual("second", chain.blocks[1].messages[0].data)
self.assertEqual("third", chain.blocks[2].messages[0].data)
self.assertIsNotNone(chain.blocks[-1].hash)
self.assertEqual(chain.blocks[1].prev_hash, chain.blocks[0].hash)
self.assertEqual(chain.blocks[2].prev_hash, chain.blocks[1].hash)
class BlockchainTestCase(TestCase):
def setUp(self):
self.blockchain = Blockchain()
def create_block(self, proof=123, previous_hash='abc'):
self.blockchain.new_block(proof, previous_hash)
def create_transaction(self, sender='a', recipient='b', amount=1):
self.blockchain.new_transaction(
sender=sender,
recipient=recipient,
amount=amount
)
def load_keys():
if wallet.load_keys():
global blockchain
blockchain = Blockchain(wallet.public_key, port)
response = {
'public_key': wallet.public_key,
'private_key': wallet.private_key,
'funds': blockchain.get_balances()
}
return jsonify(response), 201
else:
response = {
'message': 'Failed to load wallet!'
}
return jsonify(response), 500
def __init__(self, config = {}):
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.callbacks = {}
self.protocol = self.config.get('protocol','s')
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = [] # returned by interface (list from irc)
self.disconnected_servers = []
self.recent_servers = self.config.get('recent_servers',[]) # successful connections
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.server_lag = 0
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# default subscriptions
self.subscriptions = {}
self.subscriptions[self.on_banner] = [('server.banner',[])]
self.subscriptions[self.on_peers] = [('server.peers.subscribe',[])]
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.lock = Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.donation_address = ''
self.fee = None
self.relay_fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
# callbacks passed with subscriptions
self.subscriptions = defaultdict(list)
self.sub_cache = {}
# callbacks set by the GUI
self.callbacks = defaultdict(list)
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def create_keys():
wallet.create_keys()
if wallet.save_keys():
global blockchain
blockchain = Blockchain(wallet.public_key, port)
response = {
'public_key': wallet.public_key,
'private_key': wallet.private_key,
'funds': blockchain.get_balances()
}
return jsonify(response), 201
else:
response = {
'message': "Saving the keys failed."
}
return jsonify(response), 501
def __init__(self, config):
threading.Thread.__init__(self)
self.daemon = True
self.config = config
self.lock = threading.Lock()
self.blockchain = Blockchain(config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.default_server = self.config.get('server')
self.servers_list = interface.filter_protocol(interface.DEFAULT_SERVERS,'s')
self.callbacks = {}
def __init__(self, pipe, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', True)
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server('s')
self.protocol = deserialize_server(self.default_server)[2]
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.recent_servers = self.read_recent_servers()
self.pending_servers = set()
self.banner = ''
self.interface = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
self.connection_status = 'connecting'
self.requests_queue = Queue.Queue()
self.set_proxy(deserialize_proxy(self.config.get('proxy')))
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
def __init__(self, config):
threading.Thread.__init__(self)
self.daemon = True
self.config = config
self.lock = threading.Lock()
self.blockchain = Blockchain(config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.default_server = self.config.get('server')
self.disconnected_servers = []
self.callbacks = {}
self.servers = []
self.banner = ''
self.interface = None
self.heights = {}
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
host, port, protocol = self.default_server.split(':')
assert protocol in 'st'
int(port)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server('s')
self.protocol = self.default_server.split(':')[2]
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.disconnected_time = time.time()
self.recent_servers = self.config.get('recent_servers',[]) # successful connections
self.pending_servers = set()
self.banner = ''
self.interface = None
self.proxy = interface.deserialize_proxy(self.config.get('proxy'))
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# address subscriptions and cached results
self.addresses = {}
self.connection_status = 'connecting'
self.requests_queue = Queue.Queue()
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.callbacks = {}
self.protocol = self.config.get('protocol','s')
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = {} # returned by interface (list from irc)
self.pending_servers = set([])
self.disconnected_servers = set([])
self.recent_servers = self.config.get('recent_servers',[]) # successful connections
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# default subscriptions
self.subscriptions = {}
self.subscriptions[self.on_banner] = [('server.banner',[])]
self.subscriptions[self.on_peers] = [('server.peers.subscribe',[])]
self.pending_transactions_for_notifications = []
self.connection_status = 'connecting'
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get("oneserver") else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.protocol = self.config.get("protocol", "s")
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get("server")
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.disconnected_time = time.time()
self.recent_servers = self.config.get("recent_servers", []) # successful connections
self.pending_servers = set()
self.banner = ""
self.interface = None
self.proxy = self.config.get("proxy")
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join(self.config.path, "certs")
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# address subscriptions and cached results
self.addresses = {}
self.connection_status = "connecting"
self.requests_queue = Queue.Queue()
def switch_chains(self, chaincode=None):
if chaincode is None:
chain = chainparams.get_active_chain()
else:
chain = chainparams.get_chain_instance(chaincode)
self.chain_switched.clear()
self.active_chain = chain
if self.config.get_active_chain_code() != self.active_chain.code:
self.config.set_active_chain_code(self.active_chain.code)
self.print_error('switching chains to {}'.format(chain.code))
self.stop_network()
time.sleep(0.2)
self.bc_requests.clear()
self.blockchain = Blockchain(self.config, self, self.active_chain)
self.queue = Queue.Queue()
self.sanitize_default_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.interface = None
self.interfaces = {}
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
self.blockchain.init()
# Start the new network
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
time.sleep(0.2)
self.chain_switched.set()
def __init__(self, config={}):
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get("oneserver") else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.callbacks = {}
self.protocol = self.config.get("protocol", "s")
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get("server")
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = [] # returned by interface (list from irc)
self.pending_servers = set([])
self.disconnected_servers = set([])
self.recent_servers = self.config.get("recent_servers", []) # successful connections
self.banner = ""
self.interface = None
self.proxy = self.config.get("proxy")
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.server_lag = 0
dir_path = os.path.join(self.config.path, "certs")
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# default subscriptions
self.subscriptions = {}
self.subscriptions[self.on_banner] = [("server.banner", [])]
self.subscriptions[self.on_peers] = [("server.peers.subscribe", [])]
self.pending_transactions_for_notifications = []
def listen_for_input(self):
"""Starts the node and waits for user input."""
waiting_for_input = True
# A while loop for the user input interface
# It's a loop that exits once waiting_for_input becomes False or when break is called
while waiting_for_input:
print("Please enter your choice")
print("1: Add a new transaction value")
print("2: Mine Block")
print("3: Output the blockchain blocks")
print("4: Check transaction validity")
print("5: Create Wallet")
print("6: Load Wallet")
print("7: Save keys")
print("q: Quit")
user_choice = self.get_user_choice()
if user_choice == "1":
tx_data = self.get_transaction_value()
recipient, amount = tx_data
signature = self.wallet.sign_transaction(self.wallet.public_key, recipient, amount)
# Add the transaction amount to the blockchain
if self.blockchain.add_transaction(recipient, self.wallet.public_key, signature, amount=amount):
print('Added transaction!')
else:
print('Transaction failed!')
print(self.blockchain.get_open_transactions())
elif user_choice == "2":
if not self.blockchain.mine_block():
print("Mining Failed. Got no Wallet?")
elif user_choice == "3":
self.print_blockchain_blocks()
elif user_choice == "4":
# verifier = Verification()
if Verification.verify_transactions(self.blockchain.get_open_transactions(), self.blockchain.get_balances):
print("All transactions are valid!")
else:
print("There are some transactions failed!")
elif user_choice == "5":
self.wallet.create_keys()
self.blockchain = Blockchain(self.wallet.public_key)
elif user_choice == "6":
self.wallet.load_keys()
self.blockchain = Blockchain(self.wallet.public_key)
elif user_choice == "7":
self.wallet.save_keys()
elif user_choice == "q":
# This will lead to the loop to exist because it's running condition becomes False
waiting_for_input = False
else:
print("Input is invalid, please pick a value from a list.")
# verifier = Verification()
if not Verification.verify_chain(self.blockchain.get_chain()):
self.print_blockchain_blocks()
print("Invalid Blockchain!")
# Break out of the loop
break
print("Balance of {}: {:6.2f}".format(self.wallet.public_key, self.blockchain.get_balances()))
else:
print("User Left!")
print ("Done!")
class Network(threading.Thread):
def __init__(self, config):
threading.Thread.__init__(self)
self.daemon = True
self.config = config
self.lock = threading.Lock()
self.blockchain = Blockchain(config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.default_server = self.config.get('server')
self.disconnected_servers = []
self.callbacks = {}
self.servers = []
self.banner = ''
self.interface = None
self.heights = {}
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event):
with self.lock:
callbacks = self.callbacks.get(event,[])[:]
if callbacks:
[callback() for callback in callbacks]
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), 's')
for s in l:
if s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list: return
server = random.choice( choice_list )
return server
def get_servers(self):
if not self.servers:
return DEFAULT_SERVERS
else:
return self.servers
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface({'server':server})
self.interfaces[server] = i
i.start(self.queue)
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
if self.default_server:
self.start_interface(self.default_server)
self.interface = self.interfaces[self.default_server]
for i in range(8):
self.start_random_interface()
if not self.interface:
self.interface = self.interfaces.values()[0]
def start(self, wait=False):
self.start_interfaces()
threading.Thread.start(self)
if wait:
self.interface.connect_event.wait()
return self.interface.is_connected
def set_server(self, server, proxy):
i = self.interface
self.default_server = server
self.start_interface(server)
self.interface = self.interfaces[server]
i.stop_subscriptions() # fixme: it should not stop all subscriptions, and send 'unsubscribe'
self.trigger_callback('disconnecting') # for actively disconnecting
def run(self):
self.blockchain.start()
with self.lock:
self.running = True
while self.is_running():
i = self.queue.get()
if i.is_connected:
i.send([ ('blockchain.headers.subscribe',[])], self.on_header)
if i == self.interface:
i.send([('server.banner',[])], self.on_banner)
i.send([('server.peers.subscribe',[])], self.on_peers)
self.trigger_callback('connected')
else:
self.disconnected_servers.append(i.server)
self.interfaces.pop(i.server)
self.start_random_interface()
if i == self.interface:
if self.config.get('auto_cycle'):
self.interface = random.choice(self.interfaces.values())
self.config.set_key('server', self.interface.server, False)
else:
self.trigger_callback('disconnected')
def on_header(self, i, r):
result = r.get('result')
if not result: return
self.heights[i.server] = result.get('block_height')
self.blockchain.queue.put((i,result))
def on_peers(self, i, r):
if not r: return
self.servers = self.parse_servers(r.get('result'))
self.trigger_callback('peers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.trigger_callback('banner')
def stop(self):
with self.lock: self.running = False
def is_running(self):
with self.lock: return self.running
def retrieve_transaction(self, tx_hash, tx_height=0):
import transaction
r = self.interface.synchronous_get([ ('blockchain.transaction.get',[tx_hash, tx_height]) ])[0]
if r:
return transaction.Transaction(r)
def parse_servers(self, result):
""" parse servers list into dict format"""
from version import PROTOCOL_VERSION
servers = {}
for item in result:
host = item[1]
out = {}
version = None
pruning_level = '-'
if len(item) > 2:
for v in item[2]:
if re.match("[stgh]\d*", v):
protocol, port = v[0], v[1:]
if port == '': port = DEFAULT_PORTS[protocol]
out[protocol] = port
elif re.match("v(.?)+", v):
version = v[1:]
elif re.match("p\d*", v):
pruning_level = v[1:]
if pruning_level == '': pruning_level = '0'
try:
is_recent = float(version)>=float(PROTOCOL_VERSION)
except:
is_recent = False
if out and is_recent:
out['pruning'] = pruning_level
servers[host] = out
return servers
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote electrum
servers, each connected socket is handled by an Interface() object.
Connections are initiated by a Connection() thread which stops once
the connection succeeds or fails.
Our external API:
- Member functions get_header(), get_interfaces(), get_local_height(),
get_parameters(), get_server_height(), get_status_value(),
is_connected(), new_blockchain_height(), set_parameters(),
stop()
"""
def __init__(self, config=None, plugins=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.lock = Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.subscriptions = defaultdict(list)
self.callbacks = defaultdict(list)
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
self.plugins = plugins
if self.plugins:
self.plugins.set_network(self)
def register_callback(self, event, callback):
with self.lock:
self.callbacks[event].append(callback)
def trigger_callback(self, event, *args):
with self.lock:
callbacks = self.callbacks[event][:]
[callback(*args) for callback in callbacks]
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface is not None
def is_connecting(self):
return self.connection_status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def queue_request(self, method, params, interface=None):
# If you want to queue a request on any interface it must go
# through this function so message ids are properly tracked
if interface is None:
interface = self.interface
message_id = self.message_id
self.message_id += 1
if self.debug:
self.print_error(interface.host, "-->", method, params, message_id)
interface.queue_request(method, params, message_id)
return message_id
def send_subscriptions(self):
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
# Resend unanswered requests
requests = self.unanswered_requests.values()
self.unanswered_requests = {}
for request in requests:
message_id = self.queue_request(request[0], request[1])
self.unanswered_requests[message_id] = request
for addr in self.subscribed_addresses:
self.queue_request('blockchain.address.subscribe', [addr])
self.queue_request('server.banner', [])
self.queue_request('server.peers.subscribe', [])
self.queue_request('blockchain.estimatefee', [2])
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'fee':
value = self.fee
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
if key in ['status', 'updated']:
self.trigger_callback(key)
else:
self.trigger_callback(key, self.get_status_value(key))
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect
def get_interfaces(self):
'''The interfaces that are in connected state'''
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if (not server in self.interfaces and not server in self.connecting):
if server == self.default_server:
self.print_error("connecting to %s as new interface" % server)
self.set_status('connecting')
self.connecting.add(server)
c = Connection(server, self.socket_queue, self.config.path)
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
assert not self.connecting and self.socket_queue.empty()
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for interface in self.interfaces.values():
self.close_interface(interface)
assert self.interface is None
assert not self.interfaces
self.connecting = set()
# Get a new queue - no old pending connections thanks!
self.socket_queue = Queue.Queue()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server = serialize_server(host, port, protocol)
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server, True)
# abort if changes were not allowed by config
if self.config.get('server') != server or self.config.get('proxy') != proxy_str:
return
self.auto_connect = auto_connect
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
'''Switch to a random connected server other than the current one'''
servers = self.get_interfaces() # Those in connected state
if self.default_server in servers:
servers.remove(self.default_server)
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion = None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.interface = None
self.start_interface(server)
return
i = self.interfaces[server]
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.close_interface(self.interface)
self.interface = i
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def close_interface(self, interface):
if interface:
self.interfaces.pop(interface.server)
if interface.server == self.default_server:
self.interface = None
interface.close()
def add_recent_server(self, server):
# list is ordered
if server in self.recent_servers:
self.recent_servers.remove(server)
self.recent_servers.insert(0, server)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_response(self, interface, response, callback):
if self.debug:
self.print_error("<--", response)
error = response.get('error')
result = response.get('result')
method = response.get('method')
# We handle some responses; return the rest to the client.
if method == 'server.version':
interface.server_version = result
elif method == 'blockchain.headers.subscribe':
if error is None:
self.on_header(interface, result)
elif method == 'server.peers.subscribe':
if error is None:
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
if error is None:
self.banner = result
self.notify('banner')
elif method == 'blockchain.estimatefee':
if error is None:
self.fee = int(result * COIN)
self.print_error("recommended fee", self.fee)
self.notify('fee')
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(interface, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(interface, response)
else:
if callback is None:
params = response['params']
with self.lock:
for k,v in self.subscriptions.items():
if (method, params) in v:
callback = k
break
if callback is None:
self.print_error("received unexpected notification",
method, params)
else:
callback(response)
def process_responses(self, interface):
responses = interface.get_responses()
for request, response in responses:
callback = None
if request:
method, params, message_id = request
# client requests go through self.send() with a
# callback, are only sent to the current interface,
# and are placed in the unanswered_requests dictionary
client_req = self.unanswered_requests.pop(message_id, None)
if client_req:
assert interface == self.interface
callback = client_req[2]
# Copy the request method and params to the response
response['method'] = method
response['params'] = params
# Only once we've received a response to an addr subscription
# add it to the list; avoids double-sends on reconnection
if method == 'blockchain.address.subscribe':
self.subscribed_addresses.add(params[0])
else:
if not response: # Closed remotely / misbehaving
self.connection_down(interface.server)
break
# Rewrite response shape to match subscription request response
method = response.get('method')
params = response.get('params')
if method == 'blockchain.headers.subscribe':
response['result'] = params[0]
response['params'] = []
elif method == 'blockchain.address.subscribe':
response['params'] = [params[0]] # addr
response['result'] = params[1]
# Response is now in canonical form
self.process_response(interface, response, callback)
def send(self, messages, callback):
'''Messages is a list of (method, value) tuples'''
with self.lock:
self.pending_sends.append((messages, callback))
def process_pending_sends(self):
# Requests needs connectivity. If we don't have an interface,
# we cannot process them.
if not self.interface:
return
with self.lock:
sends = self.pending_sends
self.pending_sends = []
for messages, callback in sends:
subs = filter(lambda (m,v): m.endswith('.subscribe'), messages)
with self.lock:
for sub in subs:
if sub not in self.subscriptions[callback]:
self.subscriptions[callback].append(sub)
for method, params in messages:
message_id = self.queue_request(method, params)
self.unanswered_requests[message_id] = method, params, callback
def unsubscribe(self, callback):
'''Unsubscribe a callback to free object references to enable GC.'''
# Note: we can't unsubscribe from the server, so if we receive
# subsequent notifications process_response() will emit a harmless
# "received unexpected notification" warning
self.subscriptions.pop(callback, None)
def connection_down(self, server):
'''A connection to server either went down, or was never made.
We distinguish by whether it is in self.interfaces.'''
self.disconnected_servers.add(server)
if server == self.default_server:
self.set_status('disconnected')
if server in self.interfaces:
self.close_interface(self.interfaces[server])
self.heights.pop(server, None)
self.notify('interfaces')
def new_interface(self, server, socket):
self.add_recent_server(server)
self.interfaces[server] = interface = Interface(server, socket)
self.queue_request('blockchain.headers.subscribe', [], interface)
if server == self.default_server:
self.switch_to_interface(server)
self.notify('interfaces')
def maintain_sockets(self):
'''Socket maintenance.'''
# Responses to connection attempts?
while not self.socket_queue.empty():
server, socket = self.socket_queue.get()
self.connecting.remove(server)
if socket:
self.new_interface(server, socket)
else:
self.connection_down(server)
# Send pings and shut down stale interfaces
for interface in self.interfaces.values():
if interface.has_timed_out():
self.connection_down(interface.server)
elif interface.ping_required():
params = [ELECTRUM_VERSION, PROTOCOL_VERSION]
self.queue_request('server.version', params, interface)
now = time.time()
# nodes
if len(self.interfaces) + len(self.connecting) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
if not self.is_connecting():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
self.queue_request('blockchain.block.get_chunk', [idx], interface)
data['chunk_idx'] = idx
data['req_time'] = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx, response['result'])
# If not finished, get the next chunk
if idx < 0 or self.get_local_height() >= data['if_height']:
self.bc_requests.popleft()
self.notify('updated')
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
self.queue_request('blockchain.block.get_header', [height], interface)
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(data['chain'], response['result'])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify('updated')
else:
interface.print_error("header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them,
if necessary.
'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) / 2016)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object.
'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface in self.interfaces.values():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
self.connection_down(interface.server)
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def wait_on_sockets(self):
# Python docs say Windows doesn't like empty selects.
# Sleep to prevent busy looping
if not self.interfaces:
time.sleep(0.1)
return
rin = [i for i in self.interfaces.values()]
win = [i for i in self.interfaces.values() if i.unsent_requests]
rout, wout, xout = select.select(rin, win, [], 0.1)
assert not xout
for interface in wout:
interface.send_requests()
for interface in rout:
self.process_responses(interface)
def run(self):
self.blockchain.init()
while self.is_running():
self.maintain_sockets()
self.wait_on_sockets()
self.handle_bc_requests()
self.run_jobs() # Synchronizer and Verifier
self.process_pending_sends()
self.stop_network()
if self.plugins:
self.plugins.set_network(None)
self.print_error("stopped")
def on_header(self, i, header):
height = header.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get('merkle_root')
self.utxo_roots[i.server] = header.get('utxo_root')
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {'if_height': height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
def synchronous_get(self, request, timeout=100000000):
queue = Queue.Queue()
self.send([request], queue.put)
r = queue.get(True, timeout)
if r.get('error'):
raise BaseException(r.get('error'))
return r.get('result')
from itertools import chain
from flask import Flask, render_template, url_for, request, redirect, send_file
from blockchain import Blockchain
from transaction import Transaction
from zipfile38 import ZipFile
from filehandler import FileHandler
blockchain = Blockchain()
filer = FileHandler()
loggedin = None
app = Flask(__name__)
csr_cache = {}
user_returned_tokens = {}
ROOTCA = blockchain.createMiner()
@app.route("/getcert")
def getcert():
return render_template("getcert.html")
@app.route("/getcert1", methods=["POST"])
def getcert1():
name = request.form.get('name')
if name is not None:
for block in reversed(blockchain.chain):
if block.crt == 'revoke':
break
if name == block.domain:
if filer.file_exists(
filer.path_join(app.root_path, "certificates",
class Network(threading.Thread):
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.protocol = self.config.get('protocol','s')
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.disconnected_time = time.time()
self.recent_servers = self.config.get('recent_servers',[]) # successful connections
self.pending_servers = set()
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# address subscriptions and cached results
self.addresses = {}
self.connection_status = 'connecting'
self.requests_queue = Queue.Queue()
def get_server_height(self):
return self.heights.get(self.default_server,0)
def server_is_lagging(self):
h = self.get_server_height()
if not h:
print_error('no height for main interface')
return False
lag = self.get_local_height() - self.get_server_height()
return lag > 1
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected
def send_subscriptions(self):
for addr in self.addresses:
self.interface.send_request({'method':'blockchain.address.subscribe', 'params':[addr]})
self.interface.send_request({'method':'server.banner','params':[]})
self.interface.send_request({'method':'server.peers.subscribe','params':[]})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list:
return
server = random.choice( choice_list )
return server
def get_parameters(self):
host, port, protocol = self.default_server.split(':')
proxy = self.proxy
auto_connect = self.config.get('auto_cycle', True)
return host, port, protocol, proxy, auto_connect
def get_interfaces(self):
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.pending_servers.add(server)
i.start(self.queue)
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, response_queue):
self.running = True
self.response_queue = response_queue
self.start_interfaces()
t = threading.Thread(target=self.process_requests_thread)
t.daemon = True
t.start()
self.blockchain.start()
threading.Thread.start(self)
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([ host, port, protocol ])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values(): i.stop()
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected:
self.switch_to_random_interface()
else:
if self.server_is_lagging():
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
while self.interfaces:
i = random.choice(self.interfaces.values())
if i.is_connected:
self.switch_to_interface(i)
break
else:
self.remove_interface(i)
def switch_to_interface(self, interface):
server = interface.server
print_error("switching to", server)
self.interface = interface
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.set_status('connected')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected:
self.stop_interface()
# notify gui
self.set_status('connecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface( self.interfaces[server] )
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def add_interface(self, i):
self.interfaces[i.server] = i
self.notify('interfaces')
def remove_interface(self, i):
self.interfaces.pop(i.server)
self.notify('interfaces')
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
if self.server_is_lagging():
print_error( "Server is lagging", blockchain_height, self.get_server_height())
if self.config.get('auto_cycle'):
self.set_server(i.server)
self.notify('updated')
def process_response(self, i, response):
method = response['method']
if method == 'blockchain.address.subscribe':
self.on_address(i, response)
elif method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.on_peers(i, response)
elif method == 'server.banner':
self.on_banner(i, response)
else:
self.response_queue.put(response)
def process_requests_thread(self):
while self.is_running():
try:
request = self.requests_queue.get(timeout=0.1)
except Queue.Empty:
continue
self.process_request(request)
def process_request(self, request):
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
except AttributeError:
out['error'] = "unknown method"
try:
out['result'] = f(*params)
except BaseException as e:
out['error'] = str(e)
print_error("network error", str(e))
self.response_queue.put(out)
return
if method == 'blockchain.address.subscribe':
addr = params[0]
if addr in self.addresses:
self.response_queue.put({'id':_id, 'result':self.addresses[addr]})
return
self.interface.send_request(request)
def run(self):
while self.is_running():
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
if len(self.interfaces) + len(self.pending_servers) < self.num_server:
self.start_random_interface()
if not self.interfaces:
if time.time() - self.disconnected_time > DISCONNECTED_RETRY_INTERVAL:
print_error('network: retrying connections')
self.disconnected_servers = set([])
self.disconnected_time = time.time()
if not self.interface.is_connected:
if time.time() - self.disconnected_time > DISCONNECTED_RETRY_INTERVAL:
print_error("forcing reconnection")
self.queue.put((self.interface, None))
self.disconnected_time = time.time()
continue
if response is not None:
self.process_response(i, response)
continue
# if response is None it is a notification about the interface
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected:
self.add_interface(i)
self.add_recent_server(i)
i.send_request({'method':'blockchain.headers.subscribe','params':[]})
if i == self.interface:
print_error('sending subscriptions to', self.interface.server)
self.send_subscriptions()
self.set_status('connected')
else:
self.disconnected_servers.add(i.server)
if i.server in self.interfaces:
self.remove_interface(i)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.set_status('disconnected')
if not self.interface.is_connected:
if self.config.get('auto_cycle'):
self.switch_to_random_interface()
else:
if self.default_server not in self.disconnected_servers:
print_error("restarting main interface")
if self.default_server in self.interfaces.keys():
self.switch_to_interface(self.interfaces[self.default_server])
else:
self.interface = self.start_interface(self.default_server)
print_error("Network: Stopping interfaces")
for i in self.interfaces.values():
i.stop()
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
if self.server_is_lagging() and self.config.get('auto_cycle'):
print_error( "Server lagging, stopping interface")
self.stop_interface()
self.notify('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.notify('servers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.notify('banner')
def on_address(self, i, r):
addr = r.get('params')[0]
result = r.get('result')
self.addresses[addr] = result
self.response_queue.put(r)
def stop(self):
print_error("stopping network")
with self.lock:
self.running = False
def is_running(self):
with self.lock:
return self.running
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote
electrum servers, each connection is handled by its own
thread object returned from Interface(). Its external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
for r in self.unanswered_requests.values():
self.interface.send_request(r)
for addr in self.subscribed_addresses:
self.interface.send_request({'method':'blockchain.address.subscribe','params':[addr]})
self.interface.send_request({'method':'server.banner','params':[]})
self.interface.send_request({'method':'server.peers.subscribe','params':[]})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect()
def auto_connect(self):
return self.config.get('auto_connect', False)
def get_interfaces(self):
'''The interfaces that are in connected state'''
return [s for s, i in self.interfaces.items() if i.is_connected()]
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if not server in self.interfaces.keys():
if server == self.default_server:
self.set_status('connecting')
i = interface.Interface(server, self.queue, self.config)
self.interfaces[i.server] = i
i.start()
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def start(self):
self.running = True
self.blockchain.start()
util.DaemonThread.start(self)
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for i in self.interfaces.values():
i.stop()
self.interface = None
self.interfaces = {}
def set_parameters(self, host, port, protocol, proxy, auto_connect):
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
servers = self.get_interfaces() # Those in connected state
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion = None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect():
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.print_error("starting %s; will switch once connected" % server)
self.start_interface(server)
return
i = self.interfaces[server]
if not i.is_connected():
# do nothing; we will switch once connected
return
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.stop_interface()
self.interface = i
self.addr_responses = {}
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
if self.interface:
self.interface.stop()
self.interface = None
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_if_notification(self, i):
'''Handle interface addition and removal through notifications'''
if i.is_connected():
self.add_recent_server(i)
i.send_request({'method':'blockchain.headers.subscribe','params':[]})
if i.server == self.default_server:
self.switch_to_interface(i.server)
else:
self.interfaces.pop(i.server, None)
self.heights.pop(i.server, None)
if i == self.interface:
self.interface = None
self.addr_responses = {}
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
# Our set of interfaces changed
self.notify('interfaces')
def process_response(self, i, response):
# the id comes from the daemon or the network proxy
_id = response.get('id')
if _id is not None:
if i != self.interface:
return
self.unanswered_requests.pop(_id)
method = response.get('method')
result = response.get('result')
if method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
self.banner = result
self.notify('banner')
elif method == 'blockchain.address.subscribe':
addr = response.get('params')[0]
self.addr_responses[addr] = result
self.response_queue.put(response)
else:
self.response_queue.put(response)
def handle_requests(self):
'''Some requests require connectivity, others we handle locally in
process_request() and must do so in order to e.g. prevent the
daemon seeming unresponsive.
'''
unhandled = []
while not self.requests_queue.empty():
request = self.requests_queue.get()
if not self.process_request(request):
unhandled.append(request)
for request in unhandled:
self.requests_queue.put(request)
def process_request(self, request):
'''Returns true if the request was processed.'''
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
out['result'] = f(*params)
except AttributeError:
out['error'] = "unknown method"
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return True
if method == 'blockchain.address.subscribe':
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({'id':_id, 'result':self.addr_responses[addr]})
return True
# This request needs connectivity. If we don't have an
# interface, we cannot process it.
if not self.is_connected():
return False
self.unanswered_requests[_id] = request
self.interface.send_request(request)
return True
def check_interfaces(self):
now = time.time()
# nodes
if len(self.interfaces) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def run(self):
while self.is_running():
self.check_interfaces()
self.handle_requests()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
# if response is None it is a notification about the interface
if response is None:
self.process_if_notification(i)
else:
self.process_response(i, response)
self.stop_network()
self.print_error("stopped")
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Node:
"""The node which runs the local blockchain instance.
Attributes:
:id: The id of the node.
:blockchain: The blockchain which is run by this node.
"""
def __init__(self):
# self.id = "Hari"
self.wallet = Wallet()
self.wallet.create_keys()
self.blockchain = Blockchain(self.wallet.public_key)
def get_transaction_value(self):
""" Returns the input of the user (a new transaction amount) as a float. """
# Get the user input, transform it from a string to a float and store it in user_input
tx_recipient = input("Enter the recipient of the transaction: ")
tx_amount = float(input("Your transaction amount: "))
return (tx_recipient, tx_amount)
def get_user_choice(self):
"""Prompts the user for its choice and return it."""
user_choice = input("Your choice: ")
return user_choice
# This will output the blockchain blocks in loop
def print_blockchain_blocks(self):
""" Output all blocks of the blockchain. """
# Output the blockchain list to the console
for block in self.blockchain.get_chain():
print(block)
else:
print("-" * 30)
def listen_for_input(self):
"""Starts the node and waits for user input."""
waiting_for_input = True
# A while loop for the user input interface
# It's a loop that exits once waiting_for_input becomes False or when break is called
while waiting_for_input:
print("Please enter your choice")
print("1: Add a new transaction value")
print("2: Mine Block")
print("3: Output the blockchain blocks")
print("4: Check transaction validity")
print("5: Create Wallet")
print("6: Load Wallet")
print("7: Save keys")
print("q: Quit")
user_choice = self.get_user_choice()
if user_choice == "1":
tx_data = self.get_transaction_value()
recipient, amount = tx_data
signature = self.wallet.sign_transaction(self.wallet.public_key, recipient, amount)
# Add the transaction amount to the blockchain
if self.blockchain.add_transaction(recipient, self.wallet.public_key, signature, amount=amount):
print('Added transaction!')
else:
print('Transaction failed!')
print(self.blockchain.get_open_transactions())
elif user_choice == "2":
if not self.blockchain.mine_block():
print("Mining Failed. Got no Wallet?")
elif user_choice == "3":
self.print_blockchain_blocks()
elif user_choice == "4":
# verifier = Verification()
if Verification.verify_transactions(self.blockchain.get_open_transactions(), self.blockchain.get_balances):
print("All transactions are valid!")
else:
print("There are some transactions failed!")
elif user_choice == "5":
self.wallet.create_keys()
self.blockchain = Blockchain(self.wallet.public_key)
elif user_choice == "6":
self.wallet.load_keys()
self.blockchain = Blockchain(self.wallet.public_key)
elif user_choice == "7":
self.wallet.save_keys()
elif user_choice == "q":
# This will lead to the loop to exist because it's running condition becomes False
waiting_for_input = False
else:
print("Input is invalid, please pick a value from a list.")
# verifier = Verification()
if not Verification.verify_chain(self.blockchain.get_chain()):
self.print_blockchain_blocks()
print("Invalid Blockchain!")
# Break out of the loop
break
print("Balance of {}: {:6.2f}".format(self.wallet.public_key, self.blockchain.get_balances()))
else:
print("User Left!")
print ("Done!")
class Network(threading.Thread):
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type(
{}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.protocol = self.config.get('protocol', 's')
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.disconnected_time = time.time()
self.recent_servers = self.config.get('recent_servers',
[]) # successful connections
self.pending_servers = set()
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# address subscriptions and cached results
self.addresses = {}
self.connection_status = 'connecting'
self.requests_queue = Queue.Queue()
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
h = self.get_server_height()
if not h:
print_error('no height for main interface')
return False
lag = self.get_local_height() - self.get_server_height()
return lag > 1
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected
def send_subscriptions(self):
for addr in self.addresses:
self.interface.send_request({
'method': 'blockchain.address.subscribe',
'params': [addr]
})
self.interface.send_request({'method': 'server.banner', 'params': []})
self.interface.send_request({
'method': 'server.peers.subscribe',
'params': []
})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({
'method': 'network.status',
'params': [key, value]
})
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys(
):
continue
else:
choice_list.append(s)
if not choice_list:
return
server = random.choice(choice_list)
return server
def get_parameters(self):
host, port, protocol = self.default_server.split(':')
proxy = self.proxy
auto_connect = self.config.get('auto_cycle', True)
return host, port, protocol, proxy, auto_connect
def get_interfaces(self):
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = {protocol: port}
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.pending_servers.add(server)
i.start(self.queue)
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, response_queue):
self.running = True
self.response_queue = response_queue
self.start_interfaces()
t = threading.Thread(target=self.process_requests_thread)
t.daemon = True
t.start()
self.blockchain.start()
threading.Thread.start(self)
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([host, port, protocol])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values():
i.stop()
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected:
self.switch_to_random_interface()
else:
if self.server_is_lagging():
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
while self.interfaces:
i = random.choice(self.interfaces.values())
if i.is_connected:
self.switch_to_interface(i)
break
else:
self.remove_interface(i)
def switch_to_interface(self, interface):
server = interface.server
print_error("switching to", server)
self.interface = interface
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.set_status('connected')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected:
self.stop_interface()
# notify gui
self.set_status('connecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface(self.interfaces[server])
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0, s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def add_interface(self, i):
self.interfaces[i.server] = i
self.notify('interfaces')
def remove_interface(self, i):
self.interfaces.pop(i.server)
self.notify('interfaces')
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
if self.server_is_lagging():
print_error("Server is lagging", blockchain_height,
self.get_server_height())
if self.config.get('auto_cycle'):
self.set_server(i.server)
self.notify('updated')
def process_response(self, i, response):
method = response['method']
if method == 'blockchain.address.subscribe':
self.on_address(i, response)
elif method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.on_peers(i, response)
elif method == 'server.banner':
self.on_banner(i, response)
else:
self.response_queue.put(response)
def process_requests_thread(self):
while self.is_running():
try:
request = self.requests_queue.get(timeout=0.1)
except Queue.Empty:
continue
self.process_request(request)
def process_request(self, request):
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id': _id}
try:
f = getattr(self, method[8:])
except AttributeError:
out['error'] = "unknown method"
try:
out['result'] = f(*params)
except BaseException as e:
out['error'] = str(e)
print_error("network error", str(e))
self.response_queue.put(out)
return
if method == 'blockchain.address.subscribe':
addr = params[0]
if addr in self.addresses:
self.response_queue.put({
'id': _id,
'result': self.addresses[addr]
})
return
self.interface.send_request(request)
def run(self):
while self.is_running():
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
if len(self.interfaces) + len(
self.pending_servers) < self.num_server:
self.start_random_interface()
if not self.interfaces:
if time.time(
) - self.disconnected_time > DISCONNECTED_RETRY_INTERVAL:
print_error('network: retrying connections')
self.disconnected_servers = set([])
self.disconnected_time = time.time()
if not self.interface.is_connected:
if time.time(
) - self.disconnected_time > DISCONNECTED_RETRY_INTERVAL:
print_error("forcing reconnection")
self.queue.put((self.interface, None))
self.disconnected_time = time.time()
continue
if response is not None:
self.process_response(i, response)
continue
# if response is None it is a notification about the interface
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected:
self.add_interface(i)
self.add_recent_server(i)
i.send_request({
'method': 'blockchain.headers.subscribe',
'params': []
})
if i == self.interface:
print_error('sending subscriptions to',
self.interface.server)
self.send_subscriptions()
self.set_status('connected')
else:
self.disconnected_servers.add(i.server)
if i.server in self.interfaces:
self.remove_interface(i)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.set_status('disconnected')
if not self.interface.is_connected:
if self.config.get('auto_cycle'):
self.switch_to_random_interface()
else:
if self.default_server not in self.disconnected_servers:
print_error("restarting main interface")
if self.default_server in self.interfaces.keys():
self.switch_to_interface(
self.interfaces[self.default_server])
else:
self.interface = self.start_interface(
self.default_server)
print_error("Network: Stopping interfaces")
for i in self.interfaces.values():
i.stop()
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i, result))
if i == self.interface:
if self.server_is_lagging() and self.config.get('auto_cycle'):
print_error("Server lagging, stopping interface")
self.stop_interface()
self.notify('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.notify('servers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.notify('banner')
def on_address(self, i, r):
addr = r.get('params')[0]
result = r.get('result')
self.addresses[addr] = result
self.response_queue.put(r)
def stop(self):
print_error("stopping network")
with self.lock:
self.running = False
def is_running(self):
with self.lock:
return self.running
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(threading.Thread):
def __init__(self, config = {}):
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.callbacks = {}
self.protocol = self.config.get('protocol','s')
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = [] # returned by interface (list from irc)
self.disconnected_servers = []
self.recent_servers = self.config.get('recent_servers',[]) # successful connections
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.server_lag = 0
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# default subscriptions
self.subscriptions = {}
self.subscriptions[self.on_banner] = [('server.banner',[])]
self.subscriptions[self.on_peers] = [('server.peers.subscribe',[])]
def is_connected(self):
return self.interface and self.interface.is_connected
def send_subscriptions(self):
for cb, sub in self.subscriptions.items():
self.interface.send(sub, cb)
def subscribe(self, messages, callback):
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in messages:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
if self.interface and self.interface.is_connected:
self.interface.send( messages, callback )
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event):
with self.lock:
callbacks = self.callbacks.get(event,[])[:]
if callbacks:
[callback() for callback in callbacks]
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list:
if not self.interfaces:
# we are probably offline, retry later
self.disconnected_servers = []
return
server = random.choice( choice_list )
return server
def get_servers(self):
out = self.irc_servers if self.irc_servers else DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.interfaces[server] = i
i.start(self.queue)
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
self.interface = self.interfaces[self.default_server]
for i in range(NUM_SERVERS):
self.start_random_interface()
if not self.interface:
self.interface = self.interfaces.values()[0]
def start(self, wait=False):
self.start_interfaces()
threading.Thread.start(self)
if wait:
self.interface.connect_event.wait()
return self.interface.is_connected
def wait_until_connected(self):
while not self.interface:
time.sleep(1)
self.interface.connect_event.wait()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([ host, port, protocol ])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values(): i.stop()
if auto_connect:
self.interface = None
return
if auto_connect:
if not self.interface:
self.switch_to_random_interface()
else:
if self.server_lag > 0:
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
if self.interfaces:
self.switch_to_interface(random.choice(self.interfaces.values()))
def switch_to_interface(self, interface):
assert self.interface is None
server = interface.server
print_error("switching to", server)
self.interface = interface
h = self.heights.get(server)
if h:
self.server_lag = self.blockchain.height() - h
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.trigger_callback('connected')
def stop_interface(self):
self.interface.stop()
self.interface = None
def set_server(self, server):
if self.default_server == server and self.interface:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface:
self.stop_interface()
# notify gui
self.trigger_callback('disconnecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface( self.interfaces[server] )
else:
self.start_interface(server)
self.interface = self.interfaces[server]
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def new_blockchain_height(self, blockchain_height, i):
print_error('new_blockchain_height')
if self.is_connected():
h = self.heights.get(self.interface.server)
if h:
self.server_lag = blockchain_height - h
if self.server_lag > 1:
print_error( "Server is lagging", blockchain_height, h)
if self.config.get('auto_cycle'):
self.set_server(i.server)
else:
print_error('no height for main interface')
self.trigger_callback('updated')
def run(self):
self.blockchain.start()
with self.lock:
self.running = True
while self.is_running():
try:
i = self.queue.get(timeout = 30 if self.interfaces else 3)
except Queue.Empty:
if len(self.interfaces) < NUM_SERVERS:
self.start_random_interface()
continue
if i.is_connected:
self.add_recent_server(i)
i.send([ ('blockchain.headers.subscribe',[])], self.on_header)
if i == self.interface:
print_error('sending subscriptions to', self.interface.server)
self.send_subscriptions()
self.trigger_callback('connected')
else:
self.disconnected_servers.append(i.server)
self.interfaces.pop(i.server)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.interface = None
self.trigger_callback('disconnected')
if self.interface is None and self.config.get('auto_cycle'):
self.switch_to_random_interface()
def on_header(self, i, r):
result = r.get('result')
if not result: return
height = result.get('block_height')
self.heights[i.server] = height
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
self.server_lag = self.blockchain.height() - height
if self.server_lag > 1 and self.config.get('auto_cycle'):
print_error( "Server lagging, stopping interface")
self.stop_interface()
self.trigger_callback('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = self.parse_servers(r.get('result'))
self.trigger_callback('peers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.trigger_callback('banner')
def stop(self):
with self.lock: self.running = False
def is_running(self):
with self.lock: return self.running
def synchronous_get(self, requests, timeout=100000000):
queue = Queue.Queue()
ids = self.interface.send(requests, lambda i,r: queue.put(r))
id2 = ids[:]
res = {}
while ids:
r = queue.get(True, timeout)
_id = r.get('id')
if _id in ids:
ids.remove(_id)
res[_id] = r.get('result')
out = []
for _id in id2:
out.append(res[_id])
return out
def retrieve_transaction(self, tx_hash, tx_height=0):
import transaction
r = self.synchronous_get([ ('blockchain.transaction.get',[tx_hash, tx_height]) ])[0]
if r:
return transaction.Transaction(r)
def parse_servers(self, result):
""" parse servers list into dict format"""
from version import PROTOCOL_VERSION
servers = {}
for item in result:
host = item[1]
out = {}
version = None
pruning_level = '-'
if len(item) > 2:
for v in item[2]:
if re.match("[stgh]\d*", v):
protocol, port = v[0], v[1:]
if port == '': port = DEFAULT_PORTS[protocol]
out[protocol] = port
elif re.match("v(.?)+", v):
version = v[1:]
elif re.match("p\d*", v):
pruning_level = v[1:]
if pruning_level == '': pruning_level = '0'
try:
is_recent = float(version)>=float(PROTOCOL_VERSION)
except:
is_recent = False
if out and is_recent:
out['pruning'] = pruning_level
servers[host] = out
return servers
def __init__(self):
# self.id = "Hari"
self.wallet = Wallet()
self.wallet.create_keys()
self.blockchain = Blockchain(self.wallet.public_key)
def __init__(self):
# self.id = str(uuid4())
self.id = 'Greg'
self.blockchain = Blockchain(self.id)
def test_malformed_nodes(self):
blockchain = Blockchain()
blockchain.register_node('http//192.168.0.1:5000')
self.assertNotIn('192.168.0.1:5000', blockchain.nodes)
def test_valid_nodes(self):
blockchain = Blockchain()
blockchain.register_node('http://192.168.0.1:5000')
self.assertIn('192.168.0.1:5000', blockchain.nodes)
class Network(util.DaemonThread):
"""The Network class manages a set of connections to remote
electrum servers, each connection is handled by its own
thread object returned from Interface(). Its external API:
- Member functions get_header(), get_parameters(), get_status_value(),
new_blockchain_height(), set_parameters(), start(),
stop()
"""
def __init__(self, pipe, config=None, active_chain=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
if active_chain is None:
active_chain = chainparams.get_active_chain()
self.active_chain = active_chain
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self, self.active_chain)
self.queue = Queue.Queue()
self.requests_queue = pipe.send_queue
self.response_queue = pipe.get_queue
# run() will block while this event is cleared
self.chain_switched = threading.Event()
self.chain_switched.set()
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.sanitize_default_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def switch_chains(self, chaincode=None):
if chaincode is None:
chain = chainparams.get_active_chain()
else:
chain = chainparams.get_chain_instance(chaincode)
self.chain_switched.clear()
self.active_chain = chain
if self.config.get_active_chain_code() != self.active_chain.code:
self.config.set_active_chain_code(self.active_chain.code)
self.print_error('switching chains to {}'.format(chain.code))
self.stop_network()
time.sleep(0.2)
self.bc_requests.clear()
self.blockchain = Blockchain(self.config, self, self.active_chain)
self.queue = Queue.Queue()
self.sanitize_default_server()
self.irc_servers = {} # returned by interface (list from irc)
self.recent_servers = self.read_recent_servers()
self.banner = ''
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.interface = None
self.interfaces = {}
# subscriptions and requests
self.subscribed_addresses = set()
# cached address status
self.addr_responses = {}
# unanswered requests
self.unanswered_requests = {}
self.blockchain.init()
# Start the new network
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
time.sleep(0.2)
self.chain_switched.set()
def sanitize_default_server(self):
"""Load the default server from config."""
self.default_server = self.config.get('server')
self.use_ssl = self.config.get('use_ssl', True)
try:
deserialize_server(self.default_server)
except Exception:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server(active_chain=self.active_chain, protocol=get_protocol_letter(self.use_ssl))
def read_recent_servers(self):
return self.config.get('recent_servers', [])
def save_recent_servers(self):
self.config.set_key('recent_servers', self.recent_servers, True)
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
self.print_error('no height for main interface')
return False
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh))
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
# clear cache
self.cached_responses = {}
self.print_error('sending subscriptions to', self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
for r in self.unanswered_requests.values():
self.interface.send_request(r)
for addr in self.subscribed_addresses:
self.interface.send_request({'method':'blockchain.address.subscribe','params':[addr]})
self.interface.send_request({'method':'server.banner','params':[]})
self.interface.send_request({'method':'server.peers.subscribe','params':[]})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.proxy, self.auto_connect()
def auto_connect(self):
return self.config.get('auto_connect', True)
def get_interfaces(self):
'''The interfaces that are in connected state'''
return [s for s, i in self.interfaces.items() if i.is_connected()]
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = self.active_chain.DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if not server in self.interfaces.keys():
if server == self.default_server:
self.set_status('connecting')
i = interface.Interface(server, self.queue, self.config)
self.interfaces[i.server] = i
i.start()
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
self.print_error('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.set_proxy(proxy)
self.start_interfaces()
def stop_network(self):
self.print_error("stopping network")
for i in self.interfaces.values():
i.stop()
self.interface = None
self.interfaces = {}
def set_parameters(self, host, port, protocol, proxy, auto_connect):
server = serialize_server(host, port, protocol)
if self.proxy != proxy or self.protocol != protocol:
# Restart the network defaulting to the given server
self.stop_network()
self.default_server = server
self.start_network(protocol, proxy)
elif self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
servers = self.get_interfaces() # Those in connected state
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion = None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect():
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.print_error("starting %s; will switch once connected" % server)
self.start_interface(server)
return
i = self.interfaces[server]
if not i.is_connected():
# do nothing; we will switch once connected
return
if self.interface != i:
self.print_error("switching to", server)
# stop any current interface in order to terminate subscriptions
self.stop_interface()
self.interface = i
self.addr_responses = {}
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
if self.interface:
self.interface.stop()
self.interface = None
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def new_blockchain_height(self, blockchain_height, i):
self.switch_lagging_interface(i.server)
self.notify('updated')
def process_if_notification(self, i):
'''Handle interface addition and removal through notifications'''
if i.is_connected():
self.add_recent_server(i)
i.send_request({'method':'blockchain.headers.subscribe','params':[]})
if i.server == self.default_server:
self.switch_to_interface(i.server)
else:
self.interfaces.pop(i.server, None)
self.heights.pop(i.server, None)
if i == self.interface:
self.interface = None
self.addr_responses = {}
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
# Our set of interfaces changed
self.notify('interfaces')
def process_response(self, i, response):
# the id comes from the daemon or the network proxy
_id = response.get('id')
if _id is not None:
if i != self.interface:
return
self.unanswered_requests.pop(_id)
method = response.get('method')
result = response.get('result')
if method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
self.banner = result
self.notify('banner')
elif method == 'blockchain.address.subscribe':
addr = response.get('params')[0]
self.addr_responses[addr] = result
self.response_queue.put(response)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(i, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(i, response)
else:
self.response_queue.put(response)
def handle_requests(self):
'''Some requests require connectivity, others we handle locally in
process_request() and must do so in order to e.g. prevent the
daemon seeming unresponsive.
'''
unhandled = []
while not self.requests_queue.empty():
request = self.requests_queue.get()
if not self.process_request(request):
unhandled.append(request)
for request in unhandled:
self.requests_queue.put(request)
def process_request(self, request):
'''Returns true if the request was processed.'''
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
out['result'] = f(*params)
except AttributeError:
out['error'] = "unknown method"
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return True
if method == 'blockchain.address.subscribe':
addr = params[0]
self.subscribed_addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({'id':_id, 'result':self.addr_responses[addr]})
return True
# This request needs connectivity. If we don't have an
# interface, we cannot process it.
if not self.is_connected():
return False
self.unanswered_requests[_id] = request
self.interface.send_request(request)
return True
def check_interfaces(self):
now = time.time()
# nodes
if len(self.interfaces) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('retrying connections to reach preferred number of interfaces ({}/{})'.format(len(self.interfaces), self.num_server))
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
interface.print_error("requesting chunk %d" % idx)
interface.send_request({'method':'blockchain.block.get_chunk', 'params':[idx]})
data['chunk_idx'] = idx
data['req_time'] = time.time()
def on_get_chunk(self, interface, response):
'''Handle receiving a chunk of block headers'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx, response['result'])
# If not finished, get the next chunk
if idx < 0 or not idx or self.get_local_height() >= data['if_height']:
self.bc_requests.popleft()
if not idx:
interface.print_error("header didn't match checkpoint, dismissing interface")
interface.stop()
else:
self.request_chunk(interface, data, idx)
def request_header(self, interface, data, height):
interface.print_error("requesting header %d" % height)
interface.send_request({'method':'blockchain.block.get_header', 'params':[height]})
data['header_height'] = height
data['req_time'] = time.time()
if not 'chain' in data:
data['chain'] = []
def on_get_header(self, interface, response):
'''Handle receiving a single block header'''
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(data['chain'], response['result'])
# If not finished, get the next header
if next_height in [True, False]:
self.bc_requests.popleft()
if next_height:
self.notify('updated')
else:
interface.print_error("header didn't connect, dismissing interface")
interface.stop()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
'''Send a request for the next header, or a chunk of them, if necessary'''
local_height, if_height = self.get_local_height(), data['if_height']
if if_height <= local_height:
return False
elif if_height > local_height + 50:
self.request_chunk(interface, data, (local_height + 1) / self.active_chain.chunk_size)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
'''Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object'''
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if not interface.is_connected():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > 10:
interface.print_error("blockchain request timed out")
interface.stop()
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def run(self):
self.blockchain.init()
while self.is_running():
# wait if we're switching chains
self.chain_switched.wait()
self.check_interfaces()
self.handle_requests()
self.handle_bc_requests()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
# if response is None it is a notification about the interface
if response is None:
self.process_if_notification(i)
else:
self.process_response(i, response)
self.stop_network()
self.print_error("stopped")
def on_header(self, i, r):
header = r.get('result')
if not header:
return
height = header.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = header.get('merkle_root')
self.utxo_roots[i.server] = header.get('utxo_root')
# Queue this interface's height for asynchronous catch-up
self.bc_requests.append((i, {'if_height': height}))
if i == self.interface:
self.switch_lagging_interface()
self.notify('updated')
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
class Network(threading.Thread):
def __init__(self, config = {}):
threading.Thread.__init__(self)
self.daemon = True
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
self.callbacks = {}
self.protocol = self.config.get('protocol','s')
self.running = False
# Server for addresses and transactions
self.default_server = self.config.get('server')
if not self.default_server:
self.default_server = pick_random_server(self.protocol)
self.irc_servers = [] # returned by interface (list from irc)
self.pending_servers = set([])
self.disconnected_servers = set([])
self.recent_servers = self.config.get('recent_servers',[]) # successful connections
self.banner = ''
self.interface = None
self.proxy = self.config.get('proxy')
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
self.server_lag = 0
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# default subscriptions
self.subscriptions = {}
self.subscriptions[self.on_banner] = [('server.banner',[])]
self.subscriptions[self.on_peers] = [('server.peers.subscribe',[])]
self.pending_transactions_for_notifications = []
def is_connected(self):
return self.interface and self.interface.is_connected
def is_up_to_date(self):
return self.interface.is_up_to_date()
def main_server(self):
return self.interface.server
def send_subscriptions(self):
for cb, sub in self.subscriptions.items():
self.interface.send(sub, cb)
def subscribe(self, messages, callback):
with self.lock:
if self.subscriptions.get(callback) is None:
self.subscriptions[callback] = []
for message in messages:
if message not in self.subscriptions[callback]:
self.subscriptions[callback].append(message)
if self.is_connected():
self.interface.send( messages, callback )
def send(self, messages, callback):
if self.is_connected():
self.interface.send( messages, callback )
return True
else:
return False
def register_callback(self, event, callback):
with self.lock:
if not self.callbacks.get(event):
self.callbacks[event] = []
self.callbacks[event].append(callback)
def trigger_callback(self, event):
with self.lock:
callbacks = self.callbacks.get(event,[])[:]
if callbacks:
[callback() for callback in callbacks]
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list:
if not self.interfaces:
# we are probably offline, retry later
self.disconnected_servers = set([])
return
server = random.choice( choice_list )
return server
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
host, port, protocol = s.split(':')
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.config)
self.pending_servers.add(server)
i.start(self.queue)
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, wait=False):
self.start_interfaces()
threading.Thread.start(self)
if wait:
return self.wait_until_connected()
def wait_until_connected(self):
"wait until connection status is known"
if self.config.get('auto_cycle'):
# self.random_server() returns None if all servers have been tried
while not self.is_connected() and self.random_server():
time.sleep(0.1)
else:
self.interface.connect_event.wait()
return self.interface.is_connected
def set_parameters(self, host, port, protocol, proxy, auto_connect):
self.config.set_key('auto_cycle', auto_connect, True)
self.config.set_key("proxy", proxy, True)
self.config.set_key("protocol", protocol, True)
server = ':'.join([ host, port, protocol ])
self.config.set_key("server", server, True)
if self.proxy != proxy or self.protocol != protocol:
self.proxy = proxy
self.protocol = protocol
for i in self.interfaces.values(): i.stop()
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected:
self.switch_to_random_interface()
else:
if self.server_lag > 0:
self.stop_interface()
else:
self.set_server(server)
def switch_to_random_interface(self):
if self.interfaces:
self.switch_to_interface(random.choice(self.interfaces.values()))
def switch_to_interface(self, interface):
assert not self.interface.is_connected
server = interface.server
print_error("switching to", server)
self.interface = interface
h = self.heights.get(server)
if h:
self.server_lag = self.blockchain.height() - h
self.config.set_key('server', server, False)
self.default_server = server
self.send_subscriptions()
self.trigger_callback('connected')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected:
return
if self.protocol != server.split(':')[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected:
self.stop_interface()
# notify gui
self.trigger_callback('disconnecting')
# start interface
self.default_server = server
self.config.set_key("server", server, True)
if server in self.interfaces.keys():
self.switch_to_interface( self.interfaces[server] )
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.config.set_key('recent_servers', self.recent_servers)
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
h = self.heights.get(self.interface.server)
if h:
self.server_lag = blockchain_height - h
if self.server_lag > 1:
print_error( "Server is lagging", blockchain_height, h)
if self.config.get('auto_cycle'):
self.set_server(i.server)
else:
print_error('no height for main interface')
self.trigger_callback('updated')
def run(self):
self.blockchain.start()
with self.lock:
self.running = True
while self.is_running():
try:
i = self.queue.get(timeout = 30 if self.interfaces else 3)
except Queue.Empty:
if len(self.interfaces) < self.num_server:
self.start_random_interface()
continue
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected:
#if i.server in self.interfaces: raise
self.interfaces[i.server] = i
self.add_recent_server(i)
i.send([ ('blockchain.headers.subscribe',[])], self.on_header)
if i == self.interface:
print_error('sending subscriptions to', self.interface.server)
self.send_subscriptions()
self.trigger_callback('connected')
else:
self.disconnected_servers.add(i.server)
if i.server in self.interfaces:
self.interfaces.pop(i.server)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
#self.interface = None
self.trigger_callback('disconnected')
if not self.interface.is_connected and self.config.get('auto_cycle'):
self.switch_to_random_interface()
def on_header(self, i, r):
result = r.get('result')
if not result: return
height = result.get('block_height')
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
self.server_lag = self.blockchain.height() - height
if self.server_lag > 1 and self.config.get('auto_cycle'):
print_error( "Server lagging, stopping interface")
self.stop_interface()
self.trigger_callback('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.trigger_callback('peers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.trigger_callback('banner')
def stop(self):
with self.lock: self.running = False
def is_running(self):
with self.lock: return self.running
def synchronous_get(self, requests, timeout=100000000):
return self.interface.synchronous_get(requests)
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()
def setUp(self):
self.blockchain = Blockchain()
class Network(util.DaemonThread):
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
util.DaemonThread.__init__(self)
self.config = SimpleConfig(config) if type(config) == type({}) else config
self.lock = threading.Lock()
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = Blockchain(self.config, self)
self.interfaces = {}
self.queue = Queue.Queue()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
self.default_server = pick_random_server('s')
self.protocol = deserialize_server(self.default_server)[2]
self.irc_servers = {} # returned by interface (list from irc)
self.disconnected_servers = set([])
self.recent_servers = self.read_recent_servers()
self.pending_servers = set()
self.banner = ''
self.interface = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
dir_path = os.path.join( self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# address subscriptions
self.addresses = set()
# cached results
self.addr_responses = {}
self.connection_status = 'connecting'
self.requests_queue = Queue.Queue()
self.set_proxy(deserialize_proxy(self.config.get('proxy')))
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
def read_recent_servers(self):
if not self.config.path:
return []
path = os.path.join(self.config.path, "recent_servers")
try:
with open(path, "r") as f:
data = f.read()
return json.loads(data)
except:
return []
def save_recent_servers(self):
if not self.config.path:
return
path = os.path.join(self.config.path, "recent_servers")
s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
try:
with open(path, "w") as f:
f.write(s)
except:
pass
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
h = self.get_server_height()
if not h:
self.print_error('no height for main interface')
return False
lag = self.get_local_height() - self.get_server_height()
return lag > 1
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface and self.interface.is_connected()
def send_subscriptions(self):
self.print_error('sending subscriptions to', self.interface.server, len(self.addresses))
for addr in self.addresses:
self.interface.send_request({'method':'blockchain.address.subscribe', 'params':[addr]})
self.interface.send_request({'method':'server.banner','params':[]})
self.interface.send_request({'method':'server.peers.subscribe','params':[]})
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
value = self.get_status_value(key)
self.response_queue.put({'method':'network.status', 'params':[key, value]})
def random_server(self):
choice_list = []
l = filter_protocol(self.get_servers(), self.protocol)
for s in l:
if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys():
continue
else:
choice_list.append(s)
if not choice_list:
return
server = random.choice( choice_list )
return server
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
auto_connect = self.config.get('auto_cycle', True)
return host, port, protocol, self.proxy, auto_connect
def get_interfaces(self):
return self.interfaces.keys()
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = DEFAULT_SERVERS
for s in self.recent_servers:
try:
host, port, protocol = deserialize_server(s)
except:
continue
if host not in out:
out[host] = { protocol:port }
return out
def start_interface(self, server):
if server in self.interfaces.keys():
return
i = interface.Interface(server, self.queue, self.config)
self.pending_servers.add(server)
i.start()
return i
def start_random_interface(self):
server = self.random_server()
if server:
self.start_interface(server)
def start_interfaces(self):
self.interface = self.start_interface(self.default_server)
for i in range(self.num_server):
self.start_random_interface()
def start(self, response_queue):
self.running = True
self.response_queue = response_queue
self.start_interfaces()
t = threading.Thread(target=self.process_requests_thread)
t.start()
self.blockchain.start()
util.DaemonThread.start(self)
def set_proxy(self, proxy):
self.proxy = proxy
if proxy:
proxy_mode = proxy_modes.index(proxy["mode"]) + 1
socks.setdefaultproxy(proxy_mode, proxy["host"], int(proxy["port"]))
socket.socket = socks.socksocket
# prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy
socket.getaddrinfo = lambda *args: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
else:
socket.socket = socket._socketobject
socket.getaddrinfo = socket._socket.getaddrinfo
def set_parameters(self, host, port, protocol, proxy, auto_connect):
if self.proxy != proxy or self.protocol != protocol:
self.print_error('restarting network')
for i in self.interfaces.values():
i.stop()
self.interfaces.pop(i.server)
self.set_proxy(proxy)
self.protocol = protocol
self.disconnected_servers = set([])
if auto_connect:
#self.interface = None
return
if auto_connect:
if not self.interface.is_connected():
self.switch_to_random_interface()
else:
if self.server_is_lagging():
self.stop_interface()
else:
server_str = serialize_server(host, port, protocol)
self.set_server(server_str)
def switch_to_random_interface(self):
while self.interfaces:
i = random.choice(self.interfaces.values())
if i.is_connected():
self.switch_to_interface(i)
break
else:
self.remove_interface(i)
def switch_to_interface(self, interface):
server = interface.server
self.print_error("switching to", server)
self.interface = interface
self.default_server = server
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def stop_interface(self):
self.interface.stop()
def set_server(self, server):
if self.default_server == server and self.interface.is_connected():
return
if self.protocol != deserialize_server(server)[2]:
return
# stop the interface in order to terminate subscriptions
if self.interface.is_connected():
self.stop_interface()
# notify gui
self.set_status('connecting')
# start interface
self.default_server = server
if server in self.interfaces.keys():
self.switch_to_interface( self.interfaces[server] )
else:
self.interface = self.start_interface(server)
def add_recent_server(self, i):
# list is ordered
s = i.server
if s in self.recent_servers:
self.recent_servers.remove(s)
self.recent_servers.insert(0,s)
self.recent_servers = self.recent_servers[0:20]
self.save_recent_servers()
def add_interface(self, i):
self.interfaces[i.server] = i
self.notify('interfaces')
def remove_interface(self, i):
self.interfaces.pop(i.server)
self.notify('interfaces')
def new_blockchain_height(self, blockchain_height, i):
if self.is_connected():
if self.server_is_lagging():
self.print_error("Server is lagging", blockchain_height, self.get_server_height())
if self.config.get('auto_cycle'):
self.set_server(i.server)
self.notify('updated')
def process_response(self, i, response):
method = response['method']
if method == 'blockchain.address.subscribe':
self.on_address(i, response)
elif method == 'blockchain.headers.subscribe':
self.on_header(i, response)
elif method == 'server.peers.subscribe':
self.on_peers(i, response)
elif method == 'server.banner':
self.on_banner(i, response)
else:
self.response_queue.put(response)
def process_requests_thread(self):
while self.is_running():
try:
request = self.requests_queue.get(timeout=0.1)
except Queue.Empty:
continue
self.process_request(request)
def process_request(self, request):
method = request['method']
params = request['params']
_id = request['id']
if method.startswith('network.'):
out = {'id':_id}
try:
f = getattr(self, method[8:])
except AttributeError:
out['error'] = "unknown method"
try:
out['result'] = f(*params)
except BaseException as e:
out['error'] = str(e)
traceback.print_exc(file=sys.stdout)
self.print_error("network error", str(e))
self.response_queue.put(out)
return
if method == 'blockchain.address.subscribe':
addr = params[0]
self.addresses.add(addr)
if addr in self.addr_responses:
self.response_queue.put({'id':_id, 'result':self.addr_responses[addr]})
return
try:
self.interface.send_request(request)
except:
# put it back in the queue
self.print_error("warning: interface not ready for", request)
self.requests_queue.put(request)
time.sleep(0.1)
def check_interfaces(self):
now = time.time()
if len(self.interfaces) + len(self.pending_servers) < self.num_server:
self.start_random_interface()
if not self.interfaces:
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
self.print_error('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
if not self.interface.is_connected():
if self.config.get('auto_cycle'):
if self.interfaces:
self.switch_to_random_interface()
else:
if self.default_server in self.interfaces.keys():
self.switch_to_interface(self.interfaces[self.default_server])
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
if self.default_server not in self.pending_servers:
self.print_error("forcing reconnection")
self.interface = self.start_interface(self.default_server)
def run(self):
while self.is_running():
self.check_interfaces()
try:
i, response = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
if response is not None:
self.process_response(i, response)
continue
# if response is None it is a notification about the interface
if i.server in self.pending_servers:
self.pending_servers.remove(i.server)
if i.is_connected():
self.add_interface(i)
self.add_recent_server(i)
i.send_request({'method':'blockchain.headers.subscribe','params':[]})
if i == self.interface:
self.send_subscriptions()
self.set_status('connected')
else:
if i.server in self.interfaces:
self.remove_interface(i)
if i.server in self.heights:
self.heights.pop(i.server)
if i == self.interface:
self.set_status('disconnected')
self.disconnected_servers.add(i.server)
self.print_error("stopping interfaces")
for i in self.interfaces.values():
i.stop()
self.print_error("stopped")
def on_header(self, i, r):
result = r.get('result')
if not result:
return
height = result.get('block_height')
if not height:
return
self.heights[i.server] = height
self.merkle_roots[i.server] = result.get('merkle_root')
self.utxo_roots[i.server] = result.get('utxo_root')
# notify blockchain about the new height
self.blockchain.queue.put((i,result))
if i == self.interface:
if self.server_is_lagging() and self.config.get('auto_cycle'):
self.print_error("Server lagging, stopping interface")
self.stop_interface()
self.notify('updated')
def on_peers(self, i, r):
if not r: return
self.irc_servers = parse_servers(r.get('result'))
self.notify('servers')
def on_banner(self, i, r):
self.banner = r.get('result')
self.notify('banner')
def on_address(self, i, r):
addr = r.get('params')[0]
result = r.get('result')
self.addr_responses[addr] = result
self.response_queue.put(r)
def get_header(self, tx_height):
return self.blockchain.read_header(tx_height)
def get_local_height(self):
return self.blockchain.height()