class LBRYumWallet(object):
def __init__(self, lbryum_path):
self.config = SimpleConfig()
self.config.set_key('chain', 'lbrycrd_main')
self.storage = WalletStorage(lbryum_path)
self.wallet = Wallet(self.storage)
self.cmd_runner = Commands(self.config, self.wallet, None)
if not self.wallet.has_seed():
seed = self.wallet.make_seed()
self.wallet.add_seed(seed, "derp")
self.wallet.create_master_keys("derp")
self.wallet.create_main_account()
self.wallet.update_password("derp", "")
self.network = Network(self.config)
self.blockchain = get_blockchain(self.config, self.network)
print self.config.get('chain'), self.blockchain
self.wallet.storage.write()
def command(self, command_name, *args, **kwargs):
cmd_runner = Commands(self.config, self.wallet, None)
cmd = known_commands[command_name]
func = getattr(cmd_runner, cmd.name)
return func(*args, **kwargs)
def generate_address(self):
address = self.wallet.create_new_address()
self.wallet.storage.write()
return address
def getWallet(path=None):
if not path:
config = SimpleConfig()
path = config.get_wallet_path()
storage = WalletStorage(path)
if not storage.file_exists:
print >> sys.stderr, "Failed to run: No wallet to migrate"
sys.exit(1)
return Wallet(storage)
def test_can_set_options_from_system_config(self):
fake_read_system = lambda: {"lbryum_path": self.lbryum_dir}
fake_read_user = lambda _: {}
read_user_dir = lambda: self.user_dir
config = SimpleConfig(options={},
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
config.set_key("lbryum_path", "c")
self.assertEqual("c", config.get("lbryum_path"))
def test_simple_config_user_config_overrides_system_config(self):
"""Options passed in user config override system config."""
fake_read_system = lambda: {"lbryum_path": self.lbryum_dir}
fake_read_user = lambda _: {"lbryum_path": "b"}
read_user_dir = lambda: self.user_dir
config = SimpleConfig(options={},
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
self.assertEqual("b", config.get("lbryum_path"))
def test_simple_config_system_config_ignored_if_portable(self):
"""If electrum is started with the "portable" flag, system
configuration is completely ignored."""
fake_read_system = lambda: {"some_key": "some_value"}
fake_read_user = lambda _: {}
read_user_dir = lambda: self.user_dir
config = SimpleConfig(options={"portable": True},
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
self.assertEqual(config.get("some_key"), None)
def test_can_set_options_set_in_user_config(self):
another_path = tempfile.mkdtemp()
fake_read_system = lambda: {}
fake_read_user = lambda _: {"lbryum_path": self.lbryum_dir}
read_user_dir = lambda: self.user_dir
config = SimpleConfig(options={},
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
config.set_key("lbryum_path", another_path)
self.assertEqual(another_path, config.get("lbryum_path"))
def test_cannot_set_options_passed_by_command_line(self):
fake_read_system = lambda: {}
fake_read_user = lambda _: {"lbryum_path": "b"}
read_user_dir = lambda: self.user_dir
config = SimpleConfig(options=self.options,
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
config.set_key("lbryum_path", "c")
self.assertEqual(self.options.get("lbryum_path"),
config.get("lbryum_path"))
def test_simple_config_user_config_is_used_if_others_arent_specified(self):
"""If no system-wide configuration and no command-line options are
specified, the user configuration is used instead."""
fake_read_system = lambda: {}
fake_read_user = lambda _: {"lbryum_path": self.lbryum_dir}
read_user_dir = lambda: self.user_dir
config = SimpleConfig(options={},
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
self.assertEqual(self.options.get("lbryum_path"),
config.get("lbryum_path"))
def run_gui(self, config_options):
config = SimpleConfig(config_options)
if self.gui:
if hasattr(self.gui, 'new_window'):
path = config.get_wallet_path()
self.gui.new_window(path, config.get('url'))
response = "ok"
else:
response = "error: current GUI does not support multiple windows"
else:
response = "Error: Electrum is running in daemon mode. Please stop the daemon first."
return response
def test_simple_config_command_line_overrides_everything(self):
"""Options passed by command line override all other configuration
sources"""
fake_read_system = lambda: {"lbryum_path": "a"}
fake_read_user = lambda _: {"lbryum_path": "b"}
read_user_dir = lambda: self.user_dir
config = SimpleConfig(options=self.options,
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
self.assertEqual(self.options.get("lbryum_path"),
config.get("lbryum_path"))
def test_can_set_options_from_system_config_if_portable(self):
"""If the "portable" flag is set, the user can overwrite system
configuration options."""
another_path = tempfile.mkdtemp()
fake_read_system = lambda: {"lbryum_path": self.lbryum_dir}
fake_read_user = lambda _: {}
read_user_dir = lambda: self.user_dir
config = SimpleConfig(options={"portable": True},
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
config.set_key("lbryum_path", another_path)
self.assertEqual(another_path, config.get("lbryum_path"))
def __init__(self, lbryum_path):
self.config = SimpleConfig()
self.config.set_key('chain', 'lbrycrd_main')
self.storage = WalletStorage(lbryum_path)
self.wallet = Wallet(self.storage)
self.cmd_runner = Commands(self.config, self.wallet, None)
if not self.wallet.has_seed():
seed = self.wallet.make_seed()
self.wallet.add_seed(seed, "derp")
self.wallet.create_master_keys("derp")
self.wallet.create_main_account()
self.wallet.update_password("derp", "")
self.network = Network(self.config)
self.blockchain = get_blockchain(self.config, self.network)
print self.config.get('chain'), self.blockchain
self.wallet.storage.write()
def test_user_config_is_not_written_with_read_only_config(self):
"""The user config does not contain command-line options or system
options when saved."""
fake_read_system = lambda: {"something": "b"}
fake_read_user = lambda _: {"something": "a"}
read_user_dir = lambda: self.user_dir
self.options.update({"something": "c"})
config = SimpleConfig(options=self.options,
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
config.save_user_config()
contents = None
with open(os.path.join(self.lbryum_dir, "config"), "r") as f:
contents = f.read()
result = ast.literal_eval(contents)
self.assertEqual({"something": "a"}, result)
def __init__(self, db_dir):
LBRYumWallet.__init__(
self, SQLiteStorage(db_dir),
SimpleConfig({
"lbryum_path": db_dir,
"wallet_path": os.path.join(db_dir, "testwallet")
}))
self.db_dir = db_dir
self.wallet_balance = Decimal(10.0)
self.total_reserved_points = Decimal(0.0)
self.queued_payments = defaultdict(Decimal)
self.network = FakeNetwork()
assert self.config.get_wallet_path() == os.path.join(
self.db_dir, "testwallet")
def test_simple_config_key_rename(self):
"""auto_cycle was renamed auto_connect"""
fake_read_system = lambda: {}
fake_read_user = lambda _: {"auto_cycle": True}
read_user_dir = lambda: self.user_dir
config = SimpleConfig(options=self.options,
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
self.assertEqual(config.get("auto_connect"), True)
self.assertEqual(config.get("auto_cycle"), None)
fake_read_user = lambda _: {"auto_connect": False, "auto_cycle": True}
config = SimpleConfig(options=self.options,
read_system_config_function=fake_read_system,
read_user_config_function=fake_read_user,
read_user_dir_function=read_user_dir)
self.assertEqual(config.get("auto_connect"), False)
self.assertEqual(config.get("auto_cycle"), None)
def run_cmdline(self, config_options):
config = SimpleConfig(config_options)
cmdname = config.get('cmd')
cmd = Commands.known_commands[cmdname]
path = config.get_wallet_path()
wallet = self.load_wallet(path) if cmd.requires_wallet else None
# arguments passed to function
args = map(lambda x: config.get(x), cmd.params)
# decode json arguments
args = map(json_decode, args)
# options
args += map(lambda x: config.get(x), cmd.options)
cmd_runner = Commands(config, wallet, self.network,
password=config_options.get('password'),
new_password=config_options.get('new_password'))
func = getattr(cmd_runner, cmd.name)
result = func(*args)
return result
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
DaemonThread.__init__(self)
self.config = SimpleConfig(config) if isinstance(config, dict) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = get_blockchain(self.config, self)
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
default_servers = self.config.get('default_servers')
if not default_servers:
raise ValueError('No servers have been specified')
self.default_server = pick_random_server(default_servers)
self.lock = Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.banner = ''
self.fee = None
self.relay_fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
# catchup counter, used to track catchup progress before chain is verified and headers saved
self.catchup_progress = 0
# callbacks passed with subscriptions
self.subscriptions = defaultdict(list)
self.sub_cache = {}
self.callbacks = defaultdict(list)
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.online_servers = {}
self._set_online_servers()
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
DaemonThread.__init__(self)
self.config = SimpleConfig(config) if isinstance(config,
dict) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = get_blockchain(self.config, self)
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
default_servers = self.config.get('default_servers')
if not default_servers:
raise ValueError('No servers have been specified')
self.default_server = pick_random_server(default_servers)
self.lock = Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.banner = ''
self.fee = None
self.relay_fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
# catchup counter, used to track catchup progress before chain is verified and headers saved
self.catchup_progress = 0
# callbacks passed with subscriptions
self.subscriptions = defaultdict(list)
self.sub_cache = {}
self.callbacks = defaultdict(list)
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.online_servers = {}
self._set_online_servers()
self.start_network(
deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
class Network(DaemonThread):
"""The Network class manages a set of connections to remote lbryum
servers, each connected socket is handled by an Interface() object.
Connections are initiated by a Connection() thread which stops once
the connection succeeds or fails.
Our external API:
- Member functions get_header(), get_interfaces(), get_local_height(),
get_parameters(), get_server_height(), get_status_value(),
is_connected(), set_parameters(), stop()
"""
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
DaemonThread.__init__(self)
self.config = SimpleConfig(config) if isinstance(config,
dict) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = get_blockchain(self.config, self)
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
default_servers = self.config.get('default_servers')
if not default_servers:
raise ValueError('No servers have been specified')
self.default_server = pick_random_server(default_servers)
self.lock = Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.banner = ''
self.fee = None
self.relay_fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
# catchup counter, used to track catchup progress before chain is verified and headers saved
self.catchup_progress = 0
# callbacks passed with subscriptions
self.subscriptions = defaultdict(list)
self.sub_cache = {}
self.callbacks = defaultdict(list)
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.online_servers = {}
self._set_online_servers()
self.start_network(
deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def register_callback(self, callback, events):
with self.lock:
for event in events:
self.callbacks[event].append(callback)
def unregister_callback(self, callback):
with self.lock:
for callbacks in self.callbacks.values():
if callback in callbacks:
callbacks.remove(callback)
def trigger_callback(self, event, *args):
with self.lock:
callbacks = self.callbacks[event][:]
for callback in callbacks:
callback(event, *args)
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
log.info('no height for main interface')
return True
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
log.info('%s is lagging (%d vs %d)', self.default_server, sh, lh)
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface is not None
def is_connecting(self):
return self.connection_status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def queue_request(self, method, params, interface=None):
# If you want to queue a request on any interface it must go
# through this function so message ids are properly tracked
if interface is None:
interface = self.interface
message_id = self.message_id
self.message_id += 1
if self.debug:
log.debug('%s --> %s, %s, %s', interface.host, method, params,
message_id)
interface.queue_request(method, params, message_id)
return message_id
def send_subscriptions(self):
log.info(
'sending subscriptions to %s. Unanswered requests: %s, Subscribed addresses: %s',
self.interface.server, len(self.unanswered_requests),
len(self.subscribed_addresses))
self.sub_cache.clear()
# Resend unanswered requests
requests = self.unanswered_requests.values()
self.unanswered_requests = {}
for request in requests:
message_id = self.queue_request(request[0], request[1])
self.unanswered_requests[message_id] = request
for addr in self.subscribed_addresses:
self.queue_request('blockchain.address.subscribe', [addr])
self.queue_request('server.banner', [])
self.queue_request('server.peers.subscribe', [])
self.queue_request('blockchain.estimatefee', [2])
self.queue_request('blockchain.relayfee', [])
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'fee':
value = self.fee
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
if key in ['status', 'updated']:
self.trigger_callback(key)
else:
self.trigger_callback(key, self.get_status_value(key))
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.auto_connect
def get_interfaces(self):
"""The interfaces that are in connected state"""
return self.interfaces.keys()
# Do an initial pruning of lbryum servers that don't have the specified port open
def _set_online_servers(self):
servers = self.config.get('default_servers', {}).iteritems()
self.online_servers = {
host: ports
for host, ports in servers if is_online(host, ports)
}
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = self.online_servers
return out
def start_interface(self, server):
if server not in self.interfaces and server not in self.connecting:
if server == self.default_server:
log.info("connecting to %s as new interface", server)
self.set_status('connecting')
self.connecting.add(server)
c = Connection(server, self.socket_queue, self.config.path)
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol,
exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
assert not self.connecting and self.socket_queue.empty()
log.info('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.start_interfaces()
def stop_network(self):
log.info("stopping network")
for interface in self.interfaces.values():
self.close_interface(interface)
assert self.interface is None
assert not self.interfaces
self.connecting = set()
# Get a new queue - no old pending connections thanks!
self.socket_queue = Queue.Queue()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server = serialize_server(host, port, protocol)
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server, True)
# abort if changes were not allowed by config
if self.config.get('server') != server or self.config.get(
'proxy') != proxy_str:
return
self.auto_connect = auto_connect
if self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
'''Switch to a random connected server other than the current one'''
servers = self.get_interfaces() # Those in connected state
if self.default_server in servers:
servers.remove(self.default_server)
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion=None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
if suggestion and self.protocol == deserialize_server(
suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.interface = None
self.start_interface(server)
return
i = self.interfaces[server]
if self.interface != i:
log.info("switching to %s", server)
# stop any current interface in order to terminate subscriptions
self.close_interface(self.interface)
self.interface = i
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def close_interface(self, interface):
if interface:
self.interfaces.pop(interface.server)
if interface.server == self.default_server:
self.interface = None
interface.close()
def process_response(self, interface, response, callbacks):
if self.debug:
log.debug("<-- %s", response)
error = response.get('error')
result = response.get('result')
method = response.get('method')
params = response.get('params')
# We handle some responses; return the rest to the client.
if method == 'server.version':
interface.server_version = result
elif method == 'blockchain.headers.subscribe':
if error is None:
self.on_header(interface, result)
elif method == 'server.peers.subscribe':
if error is None:
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
if error is None:
self.banner = result
self.notify('banner')
elif method == 'blockchain.estimatefee':
if error is None:
self.fee = int(result * COIN)
log.info("recommended fee %s", self.fee)
self.notify('fee')
elif method == 'blockchain.relayfee':
if error is None:
self.relay_fee = int(result * COIN)
log.info("relayfee %s", self.relay_fee)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(interface, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(interface, response)
for callback in callbacks:
callback(response)
def get_index(self, method, params):
""" hashable index for subscriptions and cache"""
return str(method) + (':' + str(params[0]) if params else '')
def process_responses(self, interface):
responses = interface.get_responses()
for request, response in responses:
if request:
method, params, message_id = request
k = self.get_index(method, params)
# client requests go through self.send() with a
# callback, are only sent to the current interface,
# and are placed in the unanswered_requests dictionary
client_req = self.unanswered_requests.pop(message_id, None)
if client_req:
assert interface == self.interface
callbacks = [client_req[2]]
else:
callbacks = []
# Copy the request method and params to the response
response['method'] = method
response['params'] = params
# Only once we've received a response to an addr subscription
# add it to the list; avoids double-sends on reconnection
if method == 'blockchain.address.subscribe':
self.subscribed_addresses.add(params[0])
else:
if not response: # Closed remotely / misbehaving
self.connection_down(interface.server)
break
# Rewrite response shape to match subscription request response
method = response.get('method')
params = response.get('params')
k = self.get_index(method, params)
if method == 'blockchain.headers.subscribe':
response['result'] = params[0]
response['params'] = []
elif method == 'blockchain.address.subscribe':
response['params'] = [params[0]] # addr
response['result'] = params[1]
callbacks = self.subscriptions.get(k, [])
# update cache if it's a subscription
if method.endswith('.subscribe'):
self.sub_cache[k] = response
# Response is now in canonical form
self.process_response(interface, response, callbacks)
def send(self, messages, callback):
'''Messages is a list of (method, params) tuples'''
with self.lock:
self.pending_sends.append((messages, callback))
def process_pending_sends(self):
# Requests needs connectivity. If we don't have an interface,
# we cannot process them.
if not self.interface:
return
with self.lock:
sends = self.pending_sends
self.pending_sends = []
for messages, callback in sends:
for method, params in messages:
r = None
if method.endswith('.subscribe'):
k = self.get_index(method, params)
# add callback to list
l = self.subscriptions.get(k, [])
if callback not in l:
l.append(callback)
self.subscriptions[k] = l
# check cached response for subscriptions
r = self.sub_cache.get(k)
if r is not None:
log.warning("cache hit: %s", k)
callback(r)
else:
message_id = self.queue_request(method, params)
self.unanswered_requests[
message_id] = method, params, callback
def unsubscribe(self, callback):
'''Unsubscribe a callback to free object references to enable GC.'''
# Note: we can't unsubscribe from the server, so if we receive
# subsequent notifications process_response() will emit a harmless
# "received unexpected notification" warning
with self.lock:
for v in self.subscriptions.values():
if callback in v:
v.remove(callback)
def connection_down(self, server):
'''A connection to server either went down, or was never made.
We distinguish by whether it is in self.interfaces.'''
self.disconnected_servers.add(server)
if server == self.default_server:
self.set_status('disconnected')
if server in self.interfaces:
self.close_interface(self.interfaces[server])
self.heights.pop(server, None)
self.notify('interfaces')
def new_interface(self, server, socket):
self.interfaces[server] = interface = Interface(server, socket)
self.queue_request('blockchain.headers.subscribe', [], interface)
if server == self.default_server:
self.switch_to_interface(server)
self.notify('interfaces')
def maintain_sockets(self):
'''Socket maintenance.'''
# Responses to connection attempts?
while not self.socket_queue.empty():
server, socket = self.socket_queue.get()
self.connecting.remove(server)
if socket:
self.new_interface(server, socket)
else:
self.connection_down(server)
# Send pings and shut down stale interfaces
for interface in self.interfaces.values():
if interface.has_timed_out():
self.connection_down(interface.server)
elif interface.ping_required():
params = [LBRYUM_VERSION, PROTOCOL_VERSION]
self.queue_request('server.version', params, interface)
now = time.time()
# nodes
if len(self.interfaces) + len(self.connecting) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
log.info('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
if not self.is_connecting():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
log.debug("requesting chunk %d" % idx)
self.queue_request('blockchain.block.get_chunk', [idx], interface)
data['chunk_idx'] = idx
data['req_time'] = time.time()
def _caught_up_to_interface(self, data):
return self.get_local_height() >= data['if_height']
def _need_chunk_from_interface(self, data):
return self.get_local_height() + BLOCKS_PER_CHUNK <= data['if_height']
def on_get_chunk(self, interface, response):
"""Handle receiving a chunk of block headers"""
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx,
response['result'])
if idx < 0 or self._caught_up_to_interface(data):
self.bc_requests.popleft()
self.notify('updated')
elif self._need_chunk_from_interface(data):
self.request_chunk(interface, data, idx)
else:
self.request_header(interface, data, data['if_height'])
def request_header(self, interface, data, height):
log.debug("requesting header %d" % height)
self.queue_request('blockchain.block.get_header', [height], interface)
data['header_height'] = height
data['req_time'] = time.time()
if 'chain' not in data:
data['chain'] = []
def on_get_header(self, interface, response):
"""Handle receiving a single block header"""
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(
data['chain'], response['result'])
self.catchup_progress += 1
# If not finished, get the next header
if next_height is True or next_height is False:
self.catchup_progress = 0
self.bc_requests.popleft()
if next_height:
self.switch_lagging_interface(interface.server)
self.notify('updated')
else:
log.warning(
"header didn't connect, dismissing interface")
interface.close()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
"""Send a request for the next header, or a chunk of them,
if necessary.
"""
local_height, if_height = self.get_local_height(), data['if_height']
if if_height < local_height:
return False
elif if_height > local_height + BLOCKS_PER_CHUNK:
self.request_chunk(interface, data,
(local_height + 1) / BLOCKS_PER_CHUNK)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
"""Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object.
"""
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if interface not in self.interfaces.values():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > NETWORK_TIMEOUT:
if interface.has_timed_out(
): # disconnect only if responses are really not being received
log.error("blockchain request timed out")
self.connection_down(interface.server)
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def wait_on_sockets(self):
# Python docs say Windows doesn't like empty selects.
# Sleep to prevent busy looping
if not self.interfaces:
time.sleep(0.1)
return
rin = [i for i in self.interfaces.values()]
win = [i for i in self.interfaces.values() if i.unsent_requests]
failed = False
try:
rout, wout, xout = select.select(rin, win, [], 0.2)
except select.error as (code, msg):
if code == errno.EINTR:
return
failed = True
if failed or xout:
for interface in self.interfaces.values():
self.connection_down(interface.server)
return
for interface in wout:
interface.send_requests()
for interface in rout:
self.process_responses(interface)
class HeadersComponent(Component):
component_name = HEADERS_COMPONENT
def __init__(self, component_manager):
Component.__init__(self, component_manager)
self.config = SimpleConfig(get_wallet_config())
self._downloading_headers = None
self._headers_progress_percent = None
@property
def component(self):
return self
def get_status(self):
return {} if not self._downloading_headers else {
'downloading_headers': self._downloading_headers,
'download_progress': self._headers_progress_percent
}
@defer.inlineCallbacks
def fetch_headers_from_s3(self):
local_header_size = self.local_header_file_size()
self._headers_progress_percent = 0.0
resume_header = {"Range": "bytes={}-".format(local_header_size)}
response = yield treq.get(HEADERS_URL, headers=resume_header)
final_size_after_download = response.length + local_header_size
def collector(data, h_file, start_size):
h_file.write(data)
local_size = float(h_file.tell())
final_size = float(final_size_after_download)
self._headers_progress_percent = math.ceil(
(local_size - start_size) / (final_size - start_size) * 100)
if response.code == 406: # our file is bigger
log.warning("s3 is more out of date than we are")
# should have something to download and a final length divisible by the header size
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
s3_height = (final_size_after_download / HEADER_SIZE) - 1
local_height = self.local_header_file_height()
if s3_height > local_height:
if local_header_size:
log.info("Resuming download of %i bytes from s3",
response.length)
with open(
os.path.join(self.config.path,
"blockchain_headers"),
"a+b") as headers_file:
yield treq.collect(
response, lambda d: collector(
d, headers_file, local_header_size))
else:
with open(
os.path.join(self.config.path,
"blockchain_headers"),
"wb") as headers_file:
yield treq.collect(
response, lambda d: collector(d, headers_file, 0))
log.info(
"fetched headers from s3 (s3 height: %i), now verifying integrity after download.",
s3_height)
self._check_header_file_integrity()
else:
log.warning("s3 is more out of date than we are")
else:
log.error("invalid size for headers from s3")
def local_header_file_height(self):
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
def local_header_file_size(self):
headers_path = os.path.join(self.config.path, "blockchain_headers")
if os.path.isfile(headers_path):
return os.stat(headers_path).st_size
return 0
@defer.inlineCallbacks
def get_remote_height(self, server, port):
connected = defer.Deferred()
connected.addTimeout(3, reactor, lambda *_: None)
client = StratumClient(connected)
reactor.connectTCP(server, port, client)
yield connected
remote_height = yield client.blockchain_block_get_server_height()
client.client.transport.loseConnection()
defer.returnValue(remote_height)
@defer.inlineCallbacks
def should_download_headers_from_s3(self):
if conf.settings['blockchain_name'] != "lbrycrd_main":
defer.returnValue(False)
self._check_header_file_integrity()
s3_headers_depth = conf.settings['s3_headers_depth']
if not s3_headers_depth:
defer.returnValue(False)
local_height = self.local_header_file_height()
for server_url in self.config.get('default_servers'):
port = int(self.config.get('default_servers')[server_url]['t'])
try:
remote_height = yield self.get_remote_height(server_url, port)
log.info("%s:%i height: %i, local height: %s", server_url,
port, remote_height, local_height)
if remote_height > (local_height + s3_headers_depth):
defer.returnValue(True)
except Exception as err:
log.warning("error requesting remote height from %s:%i - %s",
server_url, port, err)
defer.returnValue(False)
def _check_header_file_integrity(self):
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
if conf.settings['blockchain_name'] != "lbrycrd_main":
return
hashsum = sha256()
checksum_height, checksum = conf.settings[
'HEADERS_FILE_SHA256_CHECKSUM']
checksum_length_in_bytes = checksum_height * HEADER_SIZE
if self.local_header_file_size() < checksum_length_in_bytes:
return
headers_path = os.path.join(self.config.path, "blockchain_headers")
with open(headers_path, "rb") as headers_file:
hashsum.update(headers_file.read(checksum_length_in_bytes))
current_checksum = hashsum.hexdigest()
if current_checksum != checksum:
msg = "Expected checksum {}, got {}".format(
checksum, current_checksum)
log.warning("Wallet file corrupted, checksum mismatch. " + msg)
log.warning("Deleting header file so it can be downloaded again.")
os.unlink(headers_path)
elif (self.local_header_file_size() % HEADER_SIZE) != 0:
log.warning(
"Header file is good up to checkpoint height, but incomplete. Truncating to checkpoint."
)
with open(headers_path, "rb+") as headers_file:
headers_file.truncate(checksum_length_in_bytes)
@defer.inlineCallbacks
def start(self):
self._downloading_headers = yield self.should_download_headers_from_s3(
)
if self._downloading_headers:
try:
yield self.fetch_headers_from_s3()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self._downloading_headers = False
def stop(self):
return defer.succeed(None)
class Network(DaemonThread):
"""The Network class manages a set of connections to remote lbryum
servers, each connected socket is handled by an Interface() object.
Connections are initiated by a Connection() thread which stops once
the connection succeeds or fails.
Our external API:
- Member functions get_header(), get_interfaces(), get_local_height(),
get_parameters(), get_server_height(), get_status_value(),
is_connected(), set_parameters(), stop()
"""
def __init__(self, config=None):
if config is None:
config = {} # Do not use mutables as default values!
DaemonThread.__init__(self)
self.config = SimpleConfig(config) if isinstance(config, dict) else config
self.num_server = 8 if not self.config.get('oneserver') else 0
self.blockchain = get_blockchain(self.config, self)
# A deque of interface header requests, processed left-to-right
self.bc_requests = deque()
# Server for addresses and transactions
self.default_server = self.config.get('server')
# Sanitize default server
try:
deserialize_server(self.default_server)
except:
self.default_server = None
if not self.default_server:
default_servers = self.config.get('default_servers')
if not default_servers:
raise ValueError('No servers have been specified')
self.default_server = pick_random_server(default_servers)
self.lock = Lock()
self.pending_sends = []
self.message_id = 0
self.debug = False
self.irc_servers = {} # returned by interface (list from irc)
self.banner = ''
self.fee = None
self.relay_fee = None
self.heights = {}
self.merkle_roots = {}
self.utxo_roots = {}
# catchup counter, used to track catchup progress before chain is verified and headers saved
self.catchup_progress = 0
# callbacks passed with subscriptions
self.subscriptions = defaultdict(list)
self.sub_cache = {}
self.callbacks = defaultdict(list)
dir_path = os.path.join(self.config.path, 'certs')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# subscriptions and requests
self.subscribed_addresses = set()
# Requests from client we've not seen a response to
self.unanswered_requests = {}
# retry times
self.server_retry_time = time.time()
self.nodes_retry_time = time.time()
# kick off the network. interface is the main server we are currently
# communicating with. interfaces is the set of servers we are connecting
# to or have an ongoing connection with
self.interface = None
self.interfaces = {}
self.auto_connect = self.config.get('auto_connect', False)
self.connecting = set()
self.socket_queue = Queue.Queue()
self.online_servers = {}
self._set_online_servers()
self.start_network(deserialize_server(self.default_server)[2],
deserialize_proxy(self.config.get('proxy')))
def register_callback(self, callback, events):
with self.lock:
for event in events:
self.callbacks[event].append(callback)
def unregister_callback(self, callback):
with self.lock:
for callbacks in self.callbacks.values():
if callback in callbacks:
callbacks.remove(callback)
def trigger_callback(self, event, *args):
with self.lock:
callbacks = self.callbacks[event][:]
for callback in callbacks:
callback(event, *args)
def get_server_height(self):
return self.heights.get(self.default_server, 0)
def server_is_lagging(self):
sh = self.get_server_height()
if not sh:
log.info('no height for main interface')
return True
lh = self.get_local_height()
result = (lh - sh) > 1
if result:
log.info('%s is lagging (%d vs %d)', self.default_server, sh, lh)
return result
def set_status(self, status):
self.connection_status = status
self.notify('status')
def is_connected(self):
return self.interface is not None
def is_connecting(self):
return self.connection_status == 'connecting'
def is_up_to_date(self):
return self.unanswered_requests == {}
def queue_request(self, method, params, interface=None):
# If you want to queue a request on any interface it must go
# through this function so message ids are properly tracked
if interface is None:
interface = self.interface
message_id = self.message_id
self.message_id += 1
if self.debug:
log.debug('%s --> %s, %s, %s', interface.host, method, params, message_id)
interface.queue_request(method, params, message_id)
return message_id
def send_subscriptions(self):
log.info(
'sending subscriptions to %s. Unanswered requests: %s, Subscribed addresses: %s',
self.interface.server, len(self.unanswered_requests), len(self.subscribed_addresses))
self.sub_cache.clear()
# Resend unanswered requests
requests = self.unanswered_requests.values()
self.unanswered_requests = {}
for request in requests:
message_id = self.queue_request(request[0], request[1])
self.unanswered_requests[message_id] = request
for addr in self.subscribed_addresses:
self.queue_request('blockchain.address.subscribe', [addr])
self.queue_request('server.banner', [])
self.queue_request('server.peers.subscribe', [])
self.queue_request('blockchain.estimatefee', [2])
self.queue_request('blockchain.relayfee', [])
def get_status_value(self, key):
if key == 'status':
value = self.connection_status
elif key == 'banner':
value = self.banner
elif key == 'fee':
value = self.fee
elif key == 'updated':
value = (self.get_local_height(), self.get_server_height())
elif key == 'servers':
value = self.get_servers()
elif key == 'interfaces':
value = self.get_interfaces()
return value
def notify(self, key):
if key in ['status', 'updated']:
self.trigger_callback(key)
else:
self.trigger_callback(key, self.get_status_value(key))
def get_parameters(self):
host, port, protocol = deserialize_server(self.default_server)
return host, port, protocol, self.auto_connect
def get_interfaces(self):
"""The interfaces that are in connected state"""
return self.interfaces.keys()
# Do an initial pruning of lbryum servers that don't have the specified port open
def _set_online_servers(self):
servers = self.config.get('default_servers', {}).iteritems()
self.online_servers = {
host: ports for host, ports in servers
if is_online(host, ports)
}
def get_servers(self):
if self.irc_servers:
out = self.irc_servers
else:
out = self.online_servers
return out
def start_interface(self, server):
if server not in self.interfaces and server not in self.connecting:
if server == self.default_server:
log.info("connecting to %s as new interface", server)
self.set_status('connecting')
self.connecting.add(server)
c = Connection(server, self.socket_queue, self.config.path)
def start_random_interface(self):
exclude_set = self.disconnected_servers.union(set(self.interfaces))
server = pick_random_server(self.get_servers(), self.protocol, exclude_set)
if server:
self.start_interface(server)
def start_interfaces(self):
self.start_interface(self.default_server)
for i in range(self.num_server - 1):
self.start_random_interface()
def start_network(self, protocol, proxy):
assert not self.interface and not self.interfaces
assert not self.connecting and self.socket_queue.empty()
log.info('starting network')
self.disconnected_servers = set([])
self.protocol = protocol
self.start_interfaces()
def stop_network(self):
log.info("stopping network")
for interface in self.interfaces.values():
self.close_interface(interface)
assert self.interface is None
assert not self.interfaces
self.connecting = set()
# Get a new queue - no old pending connections thanks!
self.socket_queue = Queue.Queue()
def set_parameters(self, host, port, protocol, proxy, auto_connect):
proxy_str = serialize_proxy(proxy)
server = serialize_server(host, port, protocol)
self.config.set_key('auto_connect', auto_connect, False)
self.config.set_key("proxy", proxy_str, False)
self.config.set_key("server", server, True)
# abort if changes were not allowed by config
if self.config.get('server') != server or self.config.get('proxy') != proxy_str:
return
self.auto_connect = auto_connect
if self.default_server != server:
self.switch_to_interface(server)
else:
self.switch_lagging_interface()
def switch_to_random_interface(self):
'''Switch to a random connected server other than the current one'''
servers = self.get_interfaces() # Those in connected state
if self.default_server in servers:
servers.remove(self.default_server)
if servers:
self.switch_to_interface(random.choice(servers))
def switch_lagging_interface(self, suggestion=None):
'''If auto_connect and lagging, switch interface'''
if self.server_is_lagging() and self.auto_connect:
if suggestion and self.protocol == deserialize_server(suggestion)[2]:
self.switch_to_interface(suggestion)
else:
self.switch_to_random_interface()
def switch_to_interface(self, server):
'''Switch to server as our interface. If no connection exists nor
being opened, start a thread to connect. The actual switch will
happen on receipt of the connection notification. Do nothing
if server already is our interface.'''
self.default_server = server
if server not in self.interfaces:
self.interface = None
self.start_interface(server)
return
i = self.interfaces[server]
if self.interface != i:
log.info("switching to %s", server)
# stop any current interface in order to terminate subscriptions
self.close_interface(self.interface)
self.interface = i
self.send_subscriptions()
self.set_status('connected')
self.notify('updated')
def close_interface(self, interface):
if interface:
self.interfaces.pop(interface.server)
if interface.server == self.default_server:
self.interface = None
interface.close()
def process_response(self, interface, response, callbacks):
if self.debug:
log.debug("<-- %s", response)
error = response.get('error')
result = response.get('result')
method = response.get('method')
params = response.get('params')
# We handle some responses; return the rest to the client.
if method == 'server.version':
interface.server_version = result
elif method == 'blockchain.headers.subscribe':
if error is None:
self.on_header(interface, result)
elif method == 'server.peers.subscribe':
if error is None:
self.irc_servers = parse_servers(result)
self.notify('servers')
elif method == 'server.banner':
if error is None:
self.banner = result
self.notify('banner')
elif method == 'blockchain.estimatefee':
if error is None:
self.fee = int(result * COIN)
log.info("recommended fee %s", self.fee)
self.notify('fee')
elif method == 'blockchain.relayfee':
if error is None:
self.relay_fee = int(result * COIN)
log.info("relayfee %s", self.relay_fee)
elif method == 'blockchain.block.get_chunk':
self.on_get_chunk(interface, response)
elif method == 'blockchain.block.get_header':
self.on_get_header(interface, response)
for callback in callbacks:
callback(response)
def get_index(self, method, params):
""" hashable index for subscriptions and cache"""
return str(method) + (':' + str(params[0]) if params else '')
def process_responses(self, interface):
responses = interface.get_responses()
for request, response in responses:
if request:
method, params, message_id = request
k = self.get_index(method, params)
# client requests go through self.send() with a
# callback, are only sent to the current interface,
# and are placed in the unanswered_requests dictionary
client_req = self.unanswered_requests.pop(message_id, None)
if client_req:
assert interface == self.interface
callbacks = [client_req[2]]
else:
callbacks = []
# Copy the request method and params to the response
response['method'] = method
response['params'] = params
# Only once we've received a response to an addr subscription
# add it to the list; avoids double-sends on reconnection
if method == 'blockchain.address.subscribe':
self.subscribed_addresses.add(params[0])
else:
if not response: # Closed remotely / misbehaving
self.connection_down(interface.server)
break
# Rewrite response shape to match subscription request response
method = response.get('method')
params = response.get('params')
k = self.get_index(method, params)
if method == 'blockchain.headers.subscribe':
response['result'] = params[0]
response['params'] = []
elif method == 'blockchain.address.subscribe':
response['params'] = [params[0]] # addr
response['result'] = params[1]
callbacks = self.subscriptions.get(k, [])
# update cache if it's a subscription
if method.endswith('.subscribe'):
self.sub_cache[k] = response
# Response is now in canonical form
self.process_response(interface, response, callbacks)
def send(self, messages, callback):
'''Messages is a list of (method, params) tuples'''
with self.lock:
self.pending_sends.append((messages, callback))
def process_pending_sends(self):
# Requests needs connectivity. If we don't have an interface,
# we cannot process them.
if not self.interface:
return
with self.lock:
sends = self.pending_sends
self.pending_sends = []
for messages, callback in sends:
for method, params in messages:
r = None
if method.endswith('.subscribe'):
k = self.get_index(method, params)
# add callback to list
l = self.subscriptions.get(k, [])
if callback not in l:
l.append(callback)
self.subscriptions[k] = l
# check cached response for subscriptions
r = self.sub_cache.get(k)
if r is not None:
log.warning("cache hit: %s", k)
callback(r)
else:
message_id = self.queue_request(method, params)
self.unanswered_requests[message_id] = method, params, callback
def unsubscribe(self, callback):
'''Unsubscribe a callback to free object references to enable GC.'''
# Note: we can't unsubscribe from the server, so if we receive
# subsequent notifications process_response() will emit a harmless
# "received unexpected notification" warning
with self.lock:
for v in self.subscriptions.values():
if callback in v:
v.remove(callback)
def connection_down(self, server):
'''A connection to server either went down, or was never made.
We distinguish by whether it is in self.interfaces.'''
self.disconnected_servers.add(server)
if server == self.default_server:
self.set_status('disconnected')
if server in self.interfaces:
self.close_interface(self.interfaces[server])
self.heights.pop(server, None)
self.notify('interfaces')
def new_interface(self, server, socket):
self.interfaces[server] = interface = Interface(server, socket)
self.queue_request('blockchain.headers.subscribe', [], interface)
if server == self.default_server:
self.switch_to_interface(server)
self.notify('interfaces')
def maintain_sockets(self):
'''Socket maintenance.'''
# Responses to connection attempts?
while not self.socket_queue.empty():
server, socket = self.socket_queue.get()
self.connecting.remove(server)
if socket:
self.new_interface(server, socket)
else:
self.connection_down(server)
# Send pings and shut down stale interfaces
for interface in self.interfaces.values():
if interface.has_timed_out():
self.connection_down(interface.server)
elif interface.ping_required():
params = [LBRYUM_VERSION, PROTOCOL_VERSION]
self.queue_request('server.version', params, interface)
now = time.time()
# nodes
if len(self.interfaces) + len(self.connecting) < self.num_server:
self.start_random_interface()
if now - self.nodes_retry_time > NODES_RETRY_INTERVAL:
log.info('network: retrying connections')
self.disconnected_servers = set([])
self.nodes_retry_time = now
# main interface
if not self.is_connected():
if self.auto_connect:
if not self.is_connecting():
self.switch_to_random_interface()
else:
if self.default_server in self.disconnected_servers:
if now - self.server_retry_time > SERVER_RETRY_INTERVAL:
self.disconnected_servers.remove(self.default_server)
self.server_retry_time = now
else:
self.switch_to_interface(self.default_server)
def request_chunk(self, interface, data, idx):
log.debug("requesting chunk %d" % idx)
self.queue_request('blockchain.block.get_chunk', [idx], interface)
data['chunk_idx'] = idx
data['req_time'] = time.time()
def _caught_up_to_interface(self, data):
return self.get_local_height() >= data['if_height']
def _need_chunk_from_interface(self, data):
return self.get_local_height() + BLOCKS_PER_CHUNK <= data['if_height']
def on_get_chunk(self, interface, response):
"""Handle receiving a chunk of block headers"""
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_idx = data.get('chunk_idx')
# Ignore unsolicited chunks
if req_if == interface and req_idx == response['params'][0]:
idx = self.blockchain.connect_chunk(req_idx, response['result'])
if idx < 0 or self._caught_up_to_interface(data):
self.bc_requests.popleft()
self.notify('updated')
elif self._need_chunk_from_interface(data):
self.request_chunk(interface, data, idx)
else:
self.request_header(interface, data, data['if_height'])
def request_header(self, interface, data, height):
log.debug("requesting header %d" % height)
self.queue_request('blockchain.block.get_header', [height], interface)
data['header_height'] = height
data['req_time'] = time.time()
if 'chain' not in data:
data['chain'] = []
def on_get_header(self, interface, response):
"""Handle receiving a single block header"""
if self.bc_requests:
req_if, data = self.bc_requests[0]
req_height = data.get('header_height', -1)
# Ignore unsolicited headers
if req_if == interface and req_height == response['params'][0]:
next_height = self.blockchain.connect_header(data['chain'], response['result'])
self.catchup_progress += 1
# If not finished, get the next header
if next_height is True or next_height is False:
self.catchup_progress = 0
self.bc_requests.popleft()
if next_height:
self.switch_lagging_interface(interface.server)
self.notify('updated')
else:
log.warning("header didn't connect, dismissing interface")
interface.close()
else:
self.request_header(interface, data, next_height)
def bc_request_headers(self, interface, data):
"""Send a request for the next header, or a chunk of them,
if necessary.
"""
local_height, if_height = self.get_local_height(), data['if_height']
if if_height < local_height:
return False
elif if_height > local_height + BLOCKS_PER_CHUNK:
self.request_chunk(interface, data, (local_height + 1) / BLOCKS_PER_CHUNK)
else:
self.request_header(interface, data, if_height)
return True
def handle_bc_requests(self):
"""Work through each interface that has notified us of a new header.
Send it requests if it is ahead of our blockchain object.
"""
while self.bc_requests:
interface, data = self.bc_requests.popleft()
# If the connection was lost move on
if interface not in self.interfaces.values():
continue
req_time = data.get('req_time')
if not req_time:
# No requests sent yet. This interface has a new height.
# Request headers if it is ahead of our blockchain
if not self.bc_request_headers(interface, data):
continue
elif time.time() - req_time > NETWORK_TIMEOUT:
if interface.has_timed_out(): # disconnect only if responses are really not being received
log.error("blockchain request timed out")
self.connection_down(interface.server)
continue
# Put updated request state back at head of deque
self.bc_requests.appendleft((interface, data))
break
def wait_on_sockets(self):
# Python docs say Windows doesn't like empty selects.
# Sleep to prevent busy looping
if not self.interfaces:
time.sleep(0.1)
return
rin = [i for i in self.interfaces.values()]
win = [i for i in self.interfaces.values() if i.unsent_requests]
failed = False
try:
rout, wout, xout = select.select(rin, win, [], 0.2)
except select.error as (code, msg):
if code == errno.EINTR:
return
failed = True
if failed or xout:
for interface in self.interfaces.values():
self.connection_down(interface.server)
return
for interface in wout:
interface.send_requests()
for interface in rout:
self.process_responses(interface)
def __init__(self, component_manager):
Component.__init__(self, component_manager)
self.config = SimpleConfig(get_wallet_config())
self._downloading_headers = None
self._headers_progress_percent = None
def make_config(config=None):
if config is None:
config = {}
return SimpleConfig(config) if isinstance(config, dict) else config