def run(self):
logging.debug("Server: starting server")
util.initialize_rpc_threadpool()
self.server = grpc.server(futures.ThreadPoolExecutor(
max_workers=prefs.get_server_pool_max_threads()),
options=None)
warp_pb2_grpc.add_WarpServicer_to_server(self, self.server)
pair = auth.get_singleton().get_server_creds()
server_credentials = grpc.ssl_server_credentials((pair, ))
self.server.add_secure_port('[::]:%d' % prefs.get_port(),
server_credentials)
self.server.start()
auth.get_singleton().restart_cert_server()
self.start_zeroconf()
self.server_thread_keepalive.clear()
self.idle_emit("server-started")
# **** RUNNING ****
while not self.server_thread_keepalive.is_set():
self.server_thread_keepalive.wait(10)
# **** STOPPING ****
logging.debug("Server: stopping server")
remote_machines = list(self.remote_machines.values())
for remote in remote_machines:
self.idle_emit("remote-machine-removed", remote)
logging.debug(
"Server: Closing connection to remote machine %s (%s:%d)" %
(remote.display_hostname, remote.ip_address, remote.port))
remote.shutdown()
remote_machines = None
logging.debug("Server: stopping authentication server")
auth.get_singleton().shutdown()
logging.debug("Server: stopping discovery and advertisement")
self.zeroconf.close()
logging.debug("Server: terminating server")
self.server.stop(grace=2).wait()
self.idle_emit("shutdown-complete")
self.server = None
util.global_rpc_threadpool.shutdown(wait=True)
logging.debug("Server: server stopped")
def RequestCertificate(self, request, context):
logging.debug(
"Registration Server RPC: RequestCertificate from %s '%s'" %
(request.hostname, request.ip))
return warp_pb2.RegResponse(
locked_cert=auth.get_singleton().get_encoded_local_cert())
def __init__(self, *args, **kargs):
button_size_group = kargs.pop("button_size_group")
entry_size_group = kargs.pop("entry_size_group")
super(GroupCodeEntry, self).__init__(*args, **kargs)
self.code = auth.get_singleton().get_group_code()
self.content_widget.set_text(self.code)
entry_size_group.add_widget(self.content_widget)
self.content_widget.connect("changed", self.text_changed)
self.set_child_packing(self.content_widget, False, False, 0,
Gtk.PackType.END)
self.set_spacing(6)
self.accept_button = Gtk.Button(label=_("Set code"))
self.accept_button.show()
self.accept_button.set_sensitive(False)
self.accept_button.get_style_context().add_class("suggested-action")
self.accept_button.connect("clicked", self.apply_clicked)
button_size_group.add_widget(self.accept_button)
self.pack_end(self.accept_button, False, False, 0)
self.reorder_child(self.accept_button, 1)
def register_v2(details):
# This will block if the remote's warp udp port is closed, until either the port is unblocked
# or we tell the auth object to shutdown, in which case the request timer will cancel and return
# here immediately (with None)
logging.info("Registering with %s (%s:%d) - api version 2" % (details.hostname, details.ip_info.ip4_address, details.auth_port))
success = False
while not details.retry_keepalive.is_set():
remote_thread = threading.Thread(target=register_with_remote_thread, args=(details,), name="remote-auth-thread-%s" % id)
logging.debug("remote-registration-thread-%s-%s:%d-%s" % (details.hostname, details.ip_info.ip4_address, details.auth_port, details.ident))
remote_thread.start()
remote_thread.join()
if details.locked_cert != None:
success = auth.get_singleton().process_remote_cert(details.hostname,
details.ip_info,
details.locked_cert)
if not success:
logging.critical("Unable to register with %s (%s:%d) - api version 2"
% (details.hostname, details.ip_info.ip4_address, details.auth_port))
details.retry_keepalive.wait(10)
else:
details.retry_keepalive.set()
return True
def remove_service(self, zeroconf, _type, name):
if name == self.service_name:
return
ident = name.partition(".")[0]
auth.get_singleton().cancel_request_loop(ident)
try:
remote = self.remote_machines[ident]
except KeyError:
logging.debug(
">>> Discovery: unknown service ident (%s) reported as gone by zc."
% ident)
return
logging.debug(
">>> Discovery: service %s (%s:%d) has disappeared." %
(remote.display_hostname, remote.ip_address, remote.port))
remote.has_zc_presence = False
def retrieve_remote_cert(details):
logging.debug("Auth: Starting a new RequestLoop for '%s' (%s:%d)" % (details.hostname, details.ip_info.ip4_address, details.port))
details.request_loop = RequestLoop(details.ip_info, details.port)
data = details.request_loop.request()
if data == None:
return False
return auth.get_singleton().process_remote_cert(details.hostname,
details.ip_info,
data)
def shutdown(self):
remote_machines = list(self.remote_machines.values())
for machine in remote_machines:
self.emit_remote_machine_removed(machine)
machine.shutdown()
remote_machines = None
try:
auth.get_singleton().cert_server.stop()
self.browser.cancel()
self.zeroconf.unregister_service(self.info)
self.zeroconf.close()
except AttributeError as e:
print(e)
pass # zeroconf never started if the server never started
auth.get_singleton().clean_cert_folder()
with self.server_runlock:
self.server_runlock.notify()
def check_cert():
# This will block if the remote's warp udp port is closed, until either the port is unblocked
# or we tell the auth object to shutdown, in which case the request timer will cancel and return
# here immediately (with None)
got_cert = auth.get_singleton().retrieve_remote_cert(
ident, remote_hostname, remote_ip, info.port)
if not got_cert:
logging.critical(
">>> Discovery: unable to authenticate with %s (%s:%d)"
% (remote_hostname, remote_ip, info.port))
return False
return True
def start_zeroconf(self):
try:
logging.info("Using bundled zeroconf v%s" % zeroconf_.__version__)
except:
logging.info("Using system zeroconf v%s" % zeroconf.__version__)
self.zeroconf = Zeroconf(interfaces=[str(self.ips)])
self.service_ident = auth.get_singleton().get_ident()
self.service_name = "%s.%s" % (self.service_ident, SERVICE_TYPE)
# If this process is killed (either kill or network issue), the service
# never gets unregistered, which will prevent remotes from seeing us
# when we come back. Our first service info is to get us back on
# momentarily, and the unregister properly, so remotes get notified.
# Then we'll do it again without the flush property for the real
# connection.
init_info = ServiceInfo(SERVICE_TYPE,
self.service_name,
port=self.port,
addresses=self.ips.as_binary_list(),
properties={
'hostname': util.get_hostname(),
'type': 'flush'
})
self.zeroconf.register_service(init_info)
time.sleep(3)
self.zeroconf.unregister_service(init_info)
time.sleep(3)
self.info = ServiceInfo(SERVICE_TYPE,
self.service_name,
port=self.port,
addresses=self.ips.as_binary_list(),
properties={
'hostname': util.get_hostname(),
'api-version': config.RPC_API_VERSION,
"auth-port": str(prefs.get_auth_port()),
'type': 'real'
})
self.zeroconf.register_service(self.info)
self.browser = ServiceBrowser(self.zeroconf,
SERVICE_TYPE,
self,
addr=str(self.ips))
return False
def start_zeroconf(self):
self.zeroconf = Zeroconf()
self.service_ident = auth.get_singleton().get_ident()
self.service_name = "%s.%s" % (self.service_ident, SERVICE_TYPE)
self.info = ServiceInfo(SERVICE_TYPE,
self.service_name,
socket.inet_aton(util.get_ip()),
prefs.get_port(),
properties={'hostname': util.get_hostname()})
self.zeroconf.register_service(self.info)
return False
def start_server(self):
self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10),
options=None)
warp_pb2_grpc.add_WarpServicer_to_server(self, self.server)
pair = auth.get_singleton().get_server_creds()
server_credentials = grpc.ssl_server_credentials((pair, ))
self.server.add_secure_port('[::]:%d' % prefs.get_port(),
server_credentials)
self.server.start()
auth.get_singleton().restart_cert_server()
self.emit_server_started()
self.start_discovery_services()
with self.server_runlock:
print("Server running")
self.server_runlock.wait()
print("Server stopping")
self.server.stop(grace=2).wait()
self.emit_shutdown_complete()
self.server = None
def start_zeroconf(self):
try:
logging.info("Using bundled zeroconf v%s" % zeroconf_.__version__)
except:
logging.info("Using system zeroconf v%s" % zeroconf.__version__)
self.zeroconf = Zeroconf()
self.service_ident = auth.get_singleton().get_ident()
self.service_name = "%s.%s" % (self.service_ident, SERVICE_TYPE)
# If this process is killed (either kill ot logout), the service
# never gets unregistered, which will prevent remotes from seeing us
# when we come back. Our first service info is to get us back on
# momentarily, and the unregister properly, so remotes get notified.
# Then we'll do it again without the flush property for the real
# connection.
init_info = ServiceInfo(SERVICE_TYPE,
self.service_name,
port=prefs.get_port(),
addresses=[socket.inet_aton(util.get_ip())],
properties={
'hostname': util.get_hostname(),
'type': 'flush'
})
self.zeroconf.register_service(init_info)
time.sleep(3)
self.zeroconf.unregister_service(init_info)
time.sleep(3)
self.info = ServiceInfo(SERVICE_TYPE,
self.service_name,
port=prefs.get_port(),
addresses=[socket.inet_aton(util.get_ip())],
properties={
'hostname': util.get_hostname(),
'type': 'real'
})
self.zeroconf.register_service(self.info)
self.browser = ServiceBrowser(self.zeroconf, SERVICE_TYPE, self)
return False
def serve_cert_thread(self):
try:
server_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# server_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
server_sock.settimeout(1.0)
server_sock.bind((self.ip_info.ip4_address, self.port))
except socket.error as e:
logging.critical("Could not create udp socket for cert requests: %s" % str(e))
return
while True:
try:
data, address = server_sock.recvfrom(2000)
if data == REQUEST:
cert_data = auth.get_singleton().get_encoded_local_cert()
server_sock.sendto(cert_data, address)
except socket.timeout as e:
if self.exit:
server_sock.close()
break
def apply_clicked(self, widget, data=None):
self.code = self.content_widget.get_text()
self.accept_button.set_sensitive(False)
auth.get_singleton().update_group_code(self.code)
def add_service(self, zeroconf, _type, name):
info = zeroconf.get_service_info(_type, name)
if info:
ident = name.partition(".")[0]
try:
remote_hostname = info.properties[b"hostname"].decode()
except KeyError:
print(
"No hostname in service info properties. Is this an old version?"
)
return
remote_ip = socket.inet_ntoa(info.address)
if ident == self.service_ident:
return
got_cert = auth.get_singleton().retrieve_remote_cert(
remote_hostname, remote_ip, info.port)
if not got_cert:
print("Unable to authenticate with %s (%s)" %
(remote_hostname, remote_ip))
return
# print("Client %s added at %s" % (name, remote_ip))
try:
machine = self.remote_machines[ident]
# Update our connect info if it changed.
machine.hostname = remote_hostname
machine.ip_address = remote_ip
machine.port = info.port
except KeyError:
display_hostname = remote_hostname
i = 1
while True:
found = False
for key in self.remote_machines.keys():
remote_machine = self.remote_machines[key]
if remote_machine.display_hostname == display_hostname:
display_hostname = "%s[%d]" % (remote_hostname, i)
found = True
break
i += 1
if not found:
break
machine = remote.RemoteMachine(ident, remote_hostname,
display_hostname, remote_ip,
info.port, self.service_ident)
self.remote_machines[ident] = machine
machine.connect("ops-changed", self.remote_ops_changed)
self.emit_remote_machine_added(machine)
machine.start()
def start(self):
self.ping_timer.clear()
self.need_shutdown = False
self.emit_machine_info_changed(
) # Let's make sure the button doesn't have junk in it if we fail to connect.
print("++ Connecting to %s (%s)" %
(self.display_hostname, self.ip_address))
self.set_remote_status(RemoteStatus.INIT_CONNECTING)
def run_secure_loop(cert):
creds = grpc.ssl_channel_credentials(cert)
with grpc.secure_channel("%s:%d" % (self.ip_address, self.port),
creds) as channel:
future = grpc.channel_ready_future(channel)
connect_retries = 0
while not self.ping_timer.is_set():
try:
future.result(timeout=2)
self.stub = warp_pb2_grpc.WarpStub(channel)
self.connected = True
break
except grpc.FutureTimeoutError:
if connect_retries < MAX_CONNECT_RETRIES:
# print("channel ready timeout, waiting 10s")
self.ping_timer.wait(self.ping_time)
connect_retries += 1
continue
else:
self.set_remote_status(RemoteStatus.UNREACHABLE)
# print("Trying to remake channel")
future.cancel()
return True
one_ping = False
while not self.ping_timer.is_set():
try:
self.stub.Ping(void, timeout=2)
if not one_ping:
# Wait
self.set_remote_status(
RemoteStatus.AWAITING_DUPLEX)
if self.check_duplex_connection():
self.set_remote_status(RemoteStatus.ONLINE)
self.update_remote_machine_info()
self.update_remote_machine_avatar()
self.ping_time = PING_TIME
one_ping = True
except grpc.RpcError as e:
if e.code() in (grpc.StatusCode.DEADLINE_EXCEEDED,
grpc.StatusCode.UNAVAILABLE):
one_ping = False
self.set_remote_status(RemoteStatus.UNREACHABLE)
self.ping_timer.wait(self.ping_time)
return False
cert = auth.get_singleton().load_cert(self.hostname, self.ip_address)
while run_secure_loop(cert):
continue
self.connected = False
def run(self):
logging.debug("Server: starting server on %s (%s)" %
(self.ips, self.iface))
logging.info("Using api version %s" % config.RPC_API_VERSION)
logging.info("Our uuid: %s" % auth.get_singleton().get_ident())
self.remote_registrar = remote_registration.Registrar(
self.ips, self.port, self.auth_port, self.ips)
util.initialize_rpc_threadpool()
options = (('grpc.keepalive_time_ms',
10 * 1000), ('grpc.keepalive_timeout_ms', 5 * 1000),
('grpc.keepalive_permit_without_calls',
True), ('grpc.http2.max_pings_without_data', 0),
('grpc.http2.min_time_between_pings_ms', 10 * 1000),
('grpc.http2.min_ping_interval_without_data_ms', 5 * 1000))
self.server = grpc.server(futures.ThreadPoolExecutor(
max_workers=prefs.get_server_pool_max_threads()),
options=options)
warp_pb2_grpc.add_WarpServicer_to_server(self, self.server)
pair = auth.get_singleton().get_server_creds()
server_credentials = grpc.ssl_server_credentials((pair, ))
if self.ips.ip4:
self.server.add_secure_port('%s:%d' % (self.ips.ip4, self.port),
server_credentials)
# if self.ips.ip6:
# self.server.add_secure_port('%s:%d' % (self.ips.ip6, self.port),
# server_credentials)
self.server.start()
self.start_zeroconf()
self.server_thread_keepalive.clear()
self.idle_emit("server-started")
logging.info("Server: ACTIVE")
# **** RUNNING ****
while not self.server_thread_keepalive.is_set():
self.server_thread_keepalive.wait(10)
# **** STOPPING ****
self.remote_registrar.shutdown_registration_servers()
self.remote_registrar = None
logging.debug("Server: stopping discovery and advertisement")
# If the network is down, this will probably print an exception - it's ok,
# zeroconf catches it.
self.zeroconf.close()
remote_machines = list(self.remote_machines.values())
for remote in remote_machines:
self.idle_emit("remote-machine-removed", remote)
logging.debug(
"Server: Closing connection to remote machine %s (%s:%d)" %
(remote.display_hostname, remote.ips, remote.port))
remote.shutdown()
remote_machines = None
logging.debug("Server: terminating server")
self.server.stop(grace=2).wait()
self.idle_emit("shutdown-complete")
self.server = None
util.global_rpc_threadpool.shutdown(wait=True)
logging.debug("Server: server stopped")
def run_secure_loop():
logging.debug(
"Remote: Starting a new connection loop for %s (%s:%d)" %
(self.display_hostname, self.ip_address, self.port))
cert = auth.get_singleton().load_cert(self.hostname,
self.ip_address)
creds = grpc.ssl_channel_credentials(cert)
with grpc.secure_channel("%s:%d" % (self.ip_address, self.port),
creds) as channel:
future = grpc.channel_ready_future(channel)
try:
future.result(timeout=4)
self.stub = warp_pb2_grpc.WarpStub(channel)
except grpc.FutureTimeoutError:
self.set_remote_status(RemoteStatus.UNREACHABLE)
future.cancel()
if not self.ping_timer.is_set():
logging.debug(
"Remote: Unable to establish secure connection with %s (%s:%d). Trying again in %ds"
% (self.display_hostname, self.ip_address,
self.port, CHANNEL_RETRY_WAIT_TIME))
self.ping_timer.wait(CHANNEL_RETRY_WAIT_TIME)
return True # run_secure_loop()
return False # run_secure_loop()
duplex_fail_counter = 0
one_ping = False # A successful duplex response lets us finish setting things up.
while not self.ping_timer.is_set():
if self.busy:
logging.debug(
"Remote Ping: Skipping keepalive ping to %s (%s:%d) (busy)"
% (self.display_hostname, self.ip_address,
self.port))
self.busy = False
else:
try:
# t = GLib.get_monotonic_time()
logging.debug("Remote Ping: to %s (%s:%d)" %
(self.display_hostname,
self.ip_address, self.port))
self.stub.Ping(warp_pb2.LookupName(
id=self.local_ident,
readable_name=util.get_hostname()),
timeout=5)
# logging.debug("Latency: %s (%s)"
# % (util.precise_format_time_span(GLib.get_monotonic_time() - t), self.display_hostname))
if not one_ping:
self.set_remote_status(
RemoteStatus.AWAITING_DUPLEX)
if self.check_duplex_connection():
logging.debug(
"Remote: Connected to %s (%s:%d)" %
(self.display_hostname,
self.ip_address, self.port))
self.set_remote_status(RemoteStatus.ONLINE)
self.rpc_call(
self.update_remote_machine_info)
self.rpc_call(
self.update_remote_machine_avatar)
one_ping = True
else:
duplex_fail_counter += 1
if duplex_fail_counter > DUPLEX_MAX_FAILURES:
logging.debug(
"Remote: CheckDuplexConnection to %s (%s:%d) failed too many times"
% (self.display_hostname,
self.ip_address, self.port))
self.ping_timer.wait(
CHANNEL_RETRY_WAIT_TIME)
return True
except grpc.RpcError as e:
logging.debug(
"Remote: Ping failed, shutting down %s (%s:%d)"
% (self.display_hostname, self.ip_address,
self.port))
break
self.ping_timer.wait(
CONNECTED_PING_TIME if self.status ==
RemoteStatus.ONLINE else DUPLEX_WAIT_PING_TIME)
# This is reached by the RpcError break above. If the remote is still discoverable, start
# the secure loop over. This could have happened as a result of a quick disco/reconnect,
# And we don't notice until it has already come back. In this case, try a new connection.
if self.has_zc_presence and not self.ping_timer.is_set():
return True # run_secure_loop()
# The ping timer has been triggered, this is an orderly shutdown.
return False # run_secure_loop()
def remote_thread_v2(self):
self.channel_keepalive.clear()
self.emit_machine_info_changed(
) # Let's make sure the button doesn't have junk in it if we fail to connect.
logging.debug(
"Remote: Attempting to connect to %s (%s) - api version 2" %
(self.display_hostname, self.ips))
self.set_remote_status(RemoteStatus.INIT_CONNECTING)
cert = auth.get_singleton().get_cached_cert(self.hostname, self.ips)
creds = grpc.ssl_channel_credentials(cert)
def run_secure_loop():
opts = (('grpc.keepalive_time_ms',
10000), ('grpc.keepalive_timeout_ms', 5000),
('grpc.keepalive_permit_without_calls',
True), ('grpc.http2.max_pings_without_data', 0),
('grpc.http2.min_time_between_pings_ms', 10000),
('grpc.http2.min_ping_interval_without_data_ms', 5000))
with grpc.secure_channel("%s:%d" % (self.ips, self.port),
creds,
options=opts) as channel:
def channel_state_changed(state):
if state != grpc.ChannelConnectivity.READY:
# The server may have already called shutdown
try:
self.shutdown()
except:
pass
intercepted_channel = grpc.intercept_channel(
channel, interceptors.ChunkDecompressor())
future = grpc.channel_ready_future(intercepted_channel)
try:
future.result(timeout=4)
channel.subscribe(channel_state_changed)
self.stub = warp_pb2_grpc.WarpStub(intercepted_channel)
self.set_remote_status(RemoteStatus.AWAITING_DUPLEX)
duplex = self.wait_for_duplex()
duplex.result(timeout=10)
self.set_remote_status(RemoteStatus.ONLINE)
self.rpc_call(self.update_remote_machine_info)
self.rpc_call(self.update_remote_machine_avatar)
# Online loop
while not self.channel_keepalive.is_set():
self.channel_keepalive.wait(.5)
##
except Exception as e:
print("exception")
self.set_remote_status(RemoteStatus.UNREACHABLE)
if isinstance(e, grpc.FutureTimeoutError):
future.cancel()
logging.critical(
"Problem while waiting for channel - api version 2: %s"
% e)
elif isinstance(e, grpc.RpcError):
logging.critical(
"Problem while awaiting duplex response - api version 2: %s - %s"
% (e.code(), e.details()))
else:
logging.critical(
"General error with remote channel connection - api version 2: %s"
% e)
self.channel_keepalive.wait(10)
finally:
channel.unsubscribe(channel_state_changed)
while not self.channel_keepalive.is_set():
run_secure_loop()
self.set_remote_status(RemoteStatus.OFFLINE)