class Webserver(BotPlugin): def __init__(self, *args, **kwargs): self.server = None self.server_thread = None self.ssl_context = None self.test_app = TestApp(flask_app) super().__init__(*args, **kwargs) def get_configuration_template(self): return { 'HOST': '0.0.0.0', 'PORT': 3141, 'SSL': { 'enabled': False, 'host': '0.0.0.0', 'port': 3142, 'certificate': '', 'key': '' } } def check_configuration(self, configuration): # it is a pain, just assume a default config if SSL is absent or set to None if configuration.get('SSL', None) is None: configuration['SSL'] = { 'enabled': False, 'host': '0.0.0.0', 'port': 3142, 'certificate': '', 'key': '' } super().check_configuration(configuration) def activate(self): if not self.config: self.log.info('Webserver is not configured. Forbid activation') return if self.server_thread and self.server_thread.is_alive(): raise Exception( 'Invalid state, you should not have a webserver already running.' ) self.server_thread = Thread(target=self.run_server, name='Webserver Thread') self.server_thread.start() self.log.debug('Webserver started.') super().activate() def deactivate(self): if self.server is not None: self.log.info('Shutting down the internal webserver.') self.server.shutdown() self.log.info('Waiting for the webserver thread to quit.') self.server_thread.join() self.log.info('Webserver shut down correctly.') super().deactivate() def run_server(self): try: host = self.config['HOST'] port = self.config['PORT'] ssl = self.config['SSL'] self.log.info('Starting the webserver on %s:%i', host, port) ssl_context = (ssl['certificate'], ssl['key']) if ssl['enabled'] else None self.server = ThreadedWSGIServer( host, ssl['port'] if ssl_context else port, flask_app, ssl_context=ssl_context) self.server.serve_forever() self.log.debug('Webserver stopped') except KeyboardInterrupt: self.log.info('Keyboard interrupt, request a global shutdown.') self.server.shutdown() except Exception: self.log.exception('The webserver exploded.') @botcmd(template='webstatus') def webstatus(self, msg, args): """ Gives a quick status of what is mapped in the internal webserver """ return { 'rules': (((rule.rule, rule.endpoint) for rule in flask_app.url_map._rules)) } @webhook def echo(self, incoming_request): """ A simple test webhook """ self.log.debug("Your incoming request is :" + str(incoming_request)) return str(incoming_request) @botcmd(split_args_with=' ') def webhook_test(self, _, args): """ Test your webhooks from within err. The syntax is : !webhook test [relative_url] [post content] It triggers the notification and generate also a little test report. """ url = args[0] content = ' '.join(args[1:]) # try to guess the content-type of what has been passed try: # try if it is plain json loads(content) contenttype = 'application/json' except ValueError: # try if it is a form splitted = content.split('=') # noinspection PyBroadException try: payload = '='.join(splitted[1:]) loads(unquote(payload)) contenttype = 'application/x-www-form-urlencoded' except Exception as _: contenttype = 'text/plain' # dunno what it is self.log.debug('Detected your post as : %s.', contenttype) response = self.test_app.post(url, params=content, content_type=contenttype) return TEST_REPORT % (url, contenttype, response.status_code) @botcmd(admin_only=True) def generate_certificate(self, _, args): """ Generate a self-signed SSL certificate for the Webserver """ yield ( 'Generating a new private key and certificate. This could take a ' 'while if your system is slow or low on entropy') key_path = os.sep.join( (self.bot_config.BOT_DATA_DIR, "webserver_key.pem")) cert_path = os.sep.join( (self.bot_config.BOT_DATA_DIR, "webserver_certificate.pem")) make_ssl_certificate(key_path=key_path, cert_path=cert_path) yield f'Certificate successfully generated and saved in {self.bot_config.BOT_DATA_DIR}.' suggested_config = self.config suggested_config['SSL']['enabled'] = True suggested_config['SSL']['host'] = suggested_config['HOST'] suggested_config['SSL']['port'] = suggested_config['PORT'] + 1 suggested_config['SSL']['key'] = key_path suggested_config['SSL']['certificate'] = cert_path yield 'To enable SSL with this certificate, the following config is recommended:' yield f'{suggested_config!r}'
class Webserver(BotPlugin): min_err_version = VERSION # don't copy paste that for your plugin, it is just because it is a bundled plugin ! max_err_version = VERSION webserver_thread = None server = None webchat_mode = False def run_webserver(self): try: host = self.config["HOST"] port = self.config["PORT"] logging.info("Starting the webserver on %s:%i" % (host, port)) if self.webchat_mode: # EVERYTHING NEEDS TO BE IN THE SAME THREAD OTHERWISE Socket.IO barfs try: from socketio import socketio_manage from socketio.namespace import BaseNamespace from socketio.mixins import RoomsMixin, BroadcastMixin except ImportError: logging.exception("Could not start the webchat view") logging.error( """ If you intend to use the webchat view please install gevent-socketio: pip install gevent-socketio """ ) class ChatNamespace(BaseNamespace, RoomsMixin, BroadcastMixin): def on_nickname(self, nickname): self.environ.setdefault("nicknames", []).append(nickname) self.socket.session["nickname"] = nickname self.broadcast_event("announcement", "%s has connected" % nickname) self.broadcast_event("nicknames", self.environ["nicknames"]) # Just have them join a default-named room self.join("main_room") def on_user_message(self, msg): self.emit_to_room("main_room", "msg_to_room", self.socket.session["nickname"], msg) message = holder.bot.build_message(msg) message.setType("groupchat") # really important for security reasons message.setFrom(self.socket.session["nickname"] + "@" + host) message.setTo(holder.bot.jid) holder.bot.callback_message(holder.bot.conn, message) def recv_message(self, message): print "PING!!!", message @holder.flask_app.route("/") def index(): return redirect("/chat.html") @holder.flask_app.route("/socket.io/<path:path>") def run_socketio(path): socketio_manage(request.environ, {"": ChatNamespace}) encapsulating_middleware = SharedDataMiddleware( holder.flask_app, {"/": os.path.join(os.path.dirname(__file__), "web-static")} ) from socketio.server import SocketIOServer self.server = SocketIOServer( (host, port), encapsulating_middleware, namespace="socket.io", policy_server=False ) else: self.server = ThreadedWSGIServer(host, port, holder.flask_app) self.server.serve_forever() logging.debug("Webserver stopped") except KeyboardInterrupt as ki: logging.exception("Keyboard interrupt, request a global shutdown.") if isinstance(self.server, ThreadedWSGIServer): logging.info("webserver is ThreadedWSGIServer") self.server.shutdown() else: logging.info("webserver is SocketIOServer") self.server.kill() self.server = None holder.bot.shutdown() except Exception as e: logging.exception("The webserver exploded.") def get_configuration_template(self): return {"HOST": "0.0.0.0", "PORT": 3141, "EXTRA_FLASK_CONFIG": None, "WEBCHAT": False} def activate(self): if not self.config: logging.info("Webserver is not configured. Forbid activation") return if self.config["EXTRA_FLASK_CONFIG"]: holder.flask_app.config.update(self.config["EXTRA_FLASK_CONFIG"]) self.webchat_mode = self.config["WEBCHAT"] if self.webserver_thread: raise Exception("Invalid state, you should not have a webserver already running.") self.webserver_thread = Thread(target=self.run_webserver, name="Webserver Thread") self.webserver_thread.start() super(Webserver, self).activate() def shutdown(self): if isinstance(self.server, ThreadedWSGIServer): logging.info("webserver is ThreadedWSGIServer") self.server.shutdown() logging.info("Waiting for the webserver to terminate...") self.webserver_thread.join() logging.info("Webserver thread died as expected.") else: logging.info("webserver is SocketIOServer") self.server.kill() # it kills it but doesn't free the thread, I have to let it leak. [reported upstream] def deactivate(self): logging.debug("Sending signal to stop the webserver") if self.server: self.shutdown() self.webserver_thread = None self.server = None super(Webserver, self).deactivate() @botcmd(template="webstatus") def webstatus(self, mess, args): """ Gives a quick status of what is mapped in the internal webserver """ return {"rules": (((rule.rule, rule.endpoint) for rule in holder.flask_app.url_map.iter_rules()))} @botcmd(split_args_with=" ") def webhook_test(self, mess, args): """ Test your webhooks from within err. The syntax is : !webhook test [name of the endpoint] [post content] It triggers the notification and generate also a little test report. You can get the list of the currently deployed endpoints with !webstatus """ endpoint = args[0] content = " ".join(args[1:]) for rule in holder.flask_app.url_map.iter_rules(): if endpoint == rule.endpoint: with holder.flask_app.test_client() as client: logging.debug("Found the matching rule : %s" % rule.rule) generated_url = generate(rule.rule, 1).next() # generate a matching url from the pattern logging.debug("Generated URL : %s" % generated_url) # try to guess the content-type of what has been passed try: # try if it is plain json loads(content) contenttype = "application/json" except ValueError: # try if it is a form splitted = content.split("=") try: payload = "=".join(splitted[1:]) loads(urllib2.unquote(payload)) contenttype = "application/x-www-form-urlencoded" except Exception as e: contenttype = "text/plain" # dunno what it is logging.debug("Detected your post as : %s" % contenttype) response = client.post(generated_url, data=content, content_type=contenttype) return TEST_REPORT % (rule.rule, generated_url, contenttype, response.status_code) return "Could not find endpoint %s. Check with !webstatus which endpoints are deployed" % endpoint def emit_mess_to_webroom(self, mess): if not self.server or not self.webchat_mode: return if hasattr(mess, "getBody") and mess.getBody() and not mess.getBody().isspace(): content, is_html = mess_2_embeddablehtml(mess) if not is_html: content = "<pre>" + content + "</pre>" else: content = "<div>" + content + "</div>" pkt = dict(type="event", name="msg_to_room", args=(mess.getFrom().getNode(), content), endpoint="") room_name = "_main_room" for sessid, socket in self.server.sockets.iteritems(): if "rooms" not in socket.session: continue if room_name in socket.session["rooms"]: socket.send_packet(pkt) def callback_message(self, conn, mess): if mess.getFrom().getDomain() != self.config["HOST"]: # TODO FIXME this is too ugly self.emit_mess_to_webroom(mess) def callback_botmessage(self, mess): self.emit_mess_to_webroom(mess)
class Webserver(BotPlugin): min_err_version = VERSION # don't copy paste that for your plugin, it is just because it is a bundled plugin ! max_err_version = VERSION webserver_thread = None server = None webchat_mode = False def run_webserver(self): try: host = self.config['HOST'] port = self.config['PORT'] logging.info('Starting the webserver on %s:%i' % (host, port)) if self.webchat_mode: # EVERYTHING NEEDS TO BE IN THE SAME THREAD OTHERWISE Socket.IO barfs try: from socketio import socketio_manage from socketio.namespace import BaseNamespace from socketio.mixins import RoomsMixin, BroadcastMixin except ImportError: logging.exception("Could not start the webchat view") logging.error(""" If you intend to use the webchat view please install gevent-socketio: pip install gevent-socketio """) class ChatNamespace(BaseNamespace, RoomsMixin, BroadcastMixin): def on_nickname(self, nickname): self.environ.setdefault('nicknames', []).append(nickname) self.socket.session['nickname'] = nickname self.broadcast_event('announcement', '%s has connected' % nickname) self.broadcast_event('nicknames', self.environ['nicknames']) # Just have them join a default-named room self.join('main_room') def on_user_message(self, msg): self.emit_to_room('main_room', 'msg_to_room', self.socket.session['nickname'], msg) message = holder.bot.build_message(msg) message.setType('groupchat') # really important for security reasons message.setFrom(self.socket.session['nickname']+ '@'+host) message.setTo(holder.bot.jid) holder.bot.callback_message(holder.bot.conn, message) def recv_message(self, message): print "PING!!!", message @holder.flask_app.route('/') def index(): return redirect('/chat.html') @holder.flask_app.route("/socket.io/<path:path>") def run_socketio(path): socketio_manage(request.environ, {'': ChatNamespace}) holder.flask_app = SharedDataMiddleware(holder.flask_app, { '/': os.path.join(os.path.dirname(__file__), 'web-static') }) from socketio.server import SocketIOServer self.server = SocketIOServer((host, port), holder.flask_app, namespace="socket.io", policy_server=False) else: self.server = ThreadedWSGIServer(host, port, holder.flask_app) self.server.serve_forever() logging.debug('Webserver stopped') except Exception as e: logging.exception('The webserver exploded.') def get_configuration_template(self): return {'HOST': '0.0.0.0', 'PORT': 3141, 'EXTRA_FLASK_CONFIG': None, 'WEBCHAT': False} def activate(self): if not self.config: logging.info('Webserver is not configured. Forbid activation') return if self.config['EXTRA_FLASK_CONFIG']: holder.flask_app.config.update(self.config['EXTRA_FLASK_CONFIG']) self.webchat_mode = self.config['WEBCHAT'] if self.webserver_thread: raise Exception('Invalid state, you should not have a webserver already running.') self.webserver_thread = Thread(target=self.run_webserver, name='Webserver Thread') self.webserver_thread.start() super(Webserver, self).activate() def deactivate(self): logging.debug('Sending signal to stop the webserver') if self.server: if isinstance(self.server, ThreadedWSGIServer): logging.info('webserver is ThreadedWSGIServer') self.server.shutdown() logging.info('Waiting for the webserver to terminate...') self.webserver_thread.join() logging.info('Webserver thread died as expected.') else: logging.info('webserver is SocketIOServer') self.server.kill() # it kills it but doesn't free the thread, I have to let it leak. [reported upstream] self.webserver_thread = None self.server = None super(Webserver, self).deactivate() @botcmd(template='webstatus') def webstatus(self, mess, args): """ Gives a quick status of what is mapped in the internal webserver """ return {'rules': (((rule.rule, rule.endpoint) for rule in holder.flask_app.url_map.iter_rules()))} @botcmd(split_args_with=' ') def webhook_test(self, mess, args): """ Test your webhooks from within err. The syntax is : !webhook test [name of the endpoint] [post content] It triggers the notification and generate also a little test report. You can get the list of the currently deployed endpoints with !webstatus """ endpoint = args[0] content = ' '.join(args[1:]) for rule in holder.flask_app.url_map.iter_rules(): if endpoint == rule.endpoint: with holder.flask_app.test_client() as client: logging.debug('Found the matching rule : %s' % rule.rule) generated_url = generate(rule.rule, 1).next() # generate a matching url from the pattern logging.debug('Generated URL : %s' % generated_url) # try to guess the content-type of what has been passed try: # try if it is plain json simplejson.loads(content) contenttype = 'application/json' except JSONDecodeError: # try if it is a form splitted = content.split('=') try: payload = '='.join(splitted[1:]) simplejson.loads(urllib2.unquote(payload)) contenttype = 'application/x-www-form-urlencoded' except Exception as e: contenttype = 'text/plain' # dunno what it is logging.debug('Detected your post as : %s' % contenttype) response = client.post(generated_url, data=content, content_type=contenttype) return TEST_REPORT % (rule.rule, generated_url, contenttype, response.status_code) return 'Could not find endpoint %s. Check with !webstatus which endpoints are deployed' % endpoint def emit_mess_to_webroom(self, mess): if hasattr(mess, 'getBody') and mess.getBody() and not mess.getBody().isspace(): content, is_html = mess_2_embeddablehtml(mess) if not is_html: content = '<pre>' + content + '</pre>' else: content = '<div>' + content + '</div>' pkt = dict(type="event", name='msg_to_room', args=(mess.getFrom().getNode(), content), endpoint='') room_name = '_main_room' for sessid, socket in self.server.sockets.iteritems(): if 'rooms' not in socket.session: continue if room_name in socket.session['rooms']: socket.send_packet(pkt) def callback_message(self, conn, mess): if mess.getFrom().getDomain() != self.config['HOST']: # TODO FIXME this is too ugly self.emit_mess_to_webroom(mess) def callback_botmessage(self, mess): if self.webchat_mode: self.emit_mess_to_webroom(mess)
class Webserver(BotPlugin): min_err_version = VERSION # don't copy paste that for your plugin, it is just because it is a bundled plugin ! max_err_version = VERSION webserver_thread = None server = None webchat_mode = False def run_webserver(self): try: host = self.config['HOST'] port = self.config['PORT'] logging.info('Starting the webserver on %s:%i' % (host, port)) if self.webchat_mode: # EVERYTHING NEEDS TO BE IN THE SAME THREAD OTHERWISE Socket.IO barfs try: from socketio import socketio_manage from socketio.namespace import BaseNamespace from socketio.mixins import RoomsMixin, BroadcastMixin except ImportError: logging.exception("Could not start the webchat view") logging.error(""" If you intend to use the webchat view please install gevent-socketio: pip install gevent-socketio """) class ChatNamespace(BaseNamespace, RoomsMixin, BroadcastMixin): def on_nickname(self, nickname): self.environ.setdefault('nicknames', []).append(nickname) self.socket.session['nickname'] = nickname self.broadcast_event('announcement', '%s has connected' % nickname) self.broadcast_event('nicknames', self.environ['nicknames']) # Just have them join a default-named room self.join('main_room') def on_user_message(self, msg): self.emit_to_room('main_room', 'msg_to_room', self.socket.session['nickname'], msg) message = holder.bot.build_message(msg) message.setType( 'groupchat' ) # really important for security reasons message.setFrom(self.socket.session['nickname'] + '@' + host) message.setTo(holder.bot.jid) holder.bot.callback_message(holder.bot.conn, message) def recv_message(self, message): print "PING!!!", message @holder.flask_app.route('/') def index(): return redirect('/chat.html') @holder.flask_app.route("/socket.io/<path:path>") def run_socketio(path): socketio_manage(request.environ, {'': ChatNamespace}) encapsulating_middleware = SharedDataMiddleware( holder.flask_app, { '/': os.path.join(os.path.dirname(__file__), 'web-static') }) from socketio.server import SocketIOServer self.server = SocketIOServer((host, port), encapsulating_middleware, namespace="socket.io", policy_server=False) else: self.server = ThreadedWSGIServer(host, port, holder.flask_app) self.server.serve_forever() logging.debug('Webserver stopped') except KeyboardInterrupt as ki: logging.exception('Keyboard interrupt, request a global shutdown.') if isinstance(self.server, ThreadedWSGIServer): logging.info('webserver is ThreadedWSGIServer') self.server.shutdown() else: logging.info('webserver is SocketIOServer') self.server.kill() self.server = None holder.bot.shutdown() except Exception as e: logging.exception('The webserver exploded.') def get_configuration_template(self): return { 'HOST': '0.0.0.0', 'PORT': 3141, 'EXTRA_FLASK_CONFIG': None, 'WEBCHAT': False } def activate(self): if not self.config: logging.info('Webserver is not configured. Forbid activation') return if self.config['EXTRA_FLASK_CONFIG']: holder.flask_app.config.update(self.config['EXTRA_FLASK_CONFIG']) self.webchat_mode = self.config['WEBCHAT'] if self.webserver_thread: raise Exception( 'Invalid state, you should not have a webserver already running.' ) self.webserver_thread = Thread(target=self.run_webserver, name='Webserver Thread') self.webserver_thread.start() super(Webserver, self).activate() def shutdown(self): if isinstance(self.server, ThreadedWSGIServer): logging.info('webserver is ThreadedWSGIServer') self.server.shutdown() logging.info('Waiting for the webserver to terminate...') self.webserver_thread.join() logging.info('Webserver thread died as expected.') else: logging.info('webserver is SocketIOServer') self.server.kill( ) # it kills it but doesn't free the thread, I have to let it leak. [reported upstream] def deactivate(self): logging.debug('Sending signal to stop the webserver') if self.server: self.shutdown() self.webserver_thread = None self.server = None super(Webserver, self).deactivate() @botcmd(template='webstatus') def webstatus(self, mess, args): """ Gives a quick status of what is mapped in the internal webserver """ return { 'rules': (((rule.rule, rule.endpoint) for rule in holder.flask_app.url_map.iter_rules())) } @botcmd(split_args_with=' ') def webhook_test(self, mess, args): """ Test your webhooks from within err. The syntax is : !webhook test [name of the endpoint] [post content] It triggers the notification and generate also a little test report. You can get the list of the currently deployed endpoints with !webstatus """ endpoint = args[0] content = ' '.join(args[1:]) for rule in holder.flask_app.url_map.iter_rules(): if endpoint == rule.endpoint: with holder.flask_app.test_client() as client: logging.debug('Found the matching rule : %s' % rule.rule) generated_url = generate( rule.rule, 1).next() # generate a matching url from the pattern logging.debug('Generated URL : %s' % generated_url) # try to guess the content-type of what has been passed try: # try if it is plain json loads(content) contenttype = 'application/json' except ValueError: # try if it is a form splitted = content.split('=') try: payload = '='.join(splitted[1:]) loads(urllib2.unquote(payload)) contenttype = 'application/x-www-form-urlencoded' except Exception as e: contenttype = 'text/plain' # dunno what it is logging.debug('Detected your post as : %s' % contenttype) response = client.post(generated_url, data=content, content_type=contenttype) return TEST_REPORT % (rule.rule, generated_url, contenttype, response.status_code) return 'Could not find endpoint %s. Check with !webstatus which endpoints are deployed' % endpoint def emit_mess_to_webroom(self, mess): if not self.server or not self.webchat_mode: return if hasattr( mess, 'getBody') and mess.getBody() and not mess.getBody().isspace(): content, is_html = mess_2_embeddablehtml(mess) if not is_html: content = '<pre>' + content + '</pre>' else: content = '<div>' + content + '</div>' pkt = dict(type="event", name='msg_to_room', args=(mess.getFrom().getNode(), content), endpoint='') room_name = '_main_room' for sessid, socket in self.server.sockets.iteritems(): if 'rooms' not in socket.session: continue if room_name in socket.session['rooms']: socket.send_packet(pkt) def callback_message(self, conn, mess): if mess.getFrom().getDomain( ) != self.config['HOST']: # TODO FIXME this is too ugly self.emit_mess_to_webroom(mess) def callback_botmessage(self, mess): self.emit_mess_to_webroom(mess)
class Webserver(BotPlugin): def __init__(self, *args, **kwargs): self.server = None self.server_thread = None self.test_app = TestApp(flask_app) super().__init__(*args, **kwargs) def configure(self, configuration: Dict) -> None: """ Configures the plugin """ self.log.debug("Starting Config") if configuration is None: configuration = dict() # name of the channel to post in get_config_item("WEBSERVER_HTTP_PORT", configuration, default="3142") super().configure(configuration) def activate(self): if self.server_thread and self.server_thread.is_alive(): raise Exception( "Invalid state, you should not have a webserver already running." ) self.server_thread = Thread(target=self.run_server, name="Webserver Thread") self.server_thread.start() self.log.debug("Webserver started.") super().activate() def deactivate(self): if self.server is not None: self.log.info("Shutting down the internal webserver.") self.server.shutdown() self.log.info("Waiting for the webserver thread to quit.") self.server_thread.join() self.log.info("Webserver shut down correctly.") super().deactivate() def run_server(self): try: host = "127.0.0.1" port = int(self.config["WEBSERVER_HTTP_PORT"]) self.log.info("Starting the webserver on %s:%i", host, port) self.server = ThreadedWSGIServer(host, port, flask_app) self.server.serve_forever() self.log.debug("Webserver stopped") except KeyboardInterrupt: self.log.info("Keyboard interrupt, request a global shutdown.") self.server.shutdown() except Exception: self.log.exception("The webserver exploded.") @botcmd def webstatus(self, msg, args): """ Gives a quick status of what is mapped in the internal webserver """ web_server_info = f"Web server is running on port {self.config['WEBSERVER_HTTP_PORT']}.\nConfigured Rules:\n" for rule in flask_app.url_map._rules: web_server_info += f"* {rule.rule} -> {rule.endpoint}\n" return web_server_info @webhook def echo(self, incoming_request): """ A simple test webhook """ self.log.debug("Your incoming request is :" + str(incoming_request)) return str(incoming_request)
class Webserver(BotPlugin): def __init__(self, *args, **kwargs): self.server = None self.server_thread = None self.ssl_context = None self.test_app = TestApp(flask_app) super().__init__(*args, **kwargs) def get_configuration_template(self): return {'HOST': '0.0.0.0', 'PORT': 3141, 'SSL': {'enabled': False, 'host': '0.0.0.0', 'port': 3142, 'certificate': '', 'key': ''}} def check_configuration(self, configuration): # it is a pain, just assume a default config if SSL is absent or set to None if configuration.get('SSL', None) is None: configuration['SSL'] = {'enabled': False, 'host': '0.0.0.0', 'port': 3142, 'certificate': '', 'key': ''} super().check_configuration(configuration) def activate(self): if not self.config: self.log.info('Webserver is not configured. Forbid activation') return if self.server_thread and self.server_thread.is_alive(): raise Exception('Invalid state, you should not have a webserver already running.') self.server_thread = Thread(target=self.run_server, name='Webserver Thread') self.server_thread.start() self.log.debug('Webserver started.') super().activate() def deactivate(self): if self.server is not None: self.log.info('Shutting down the internal webserver.') self.server.shutdown() self.log.info('Waiting for the webserver thread to quit.') self.server_thread.join() self.log.info('Webserver shut down correctly.') super().deactivate() def run_server(self): try: host = self.config['HOST'] port = self.config['PORT'] ssl = self.config['SSL'] self.log.info('Starting the webserver on %s:%i', host, port) ssl_context = (ssl['certificate'], ssl['key']) if ssl['enabled'] else None self.server = ThreadedWSGIServer(host, ssl['port'] if ssl_context else port, flask_app, ssl_context=ssl_context) self.server.serve_forever() self.log.debug('Webserver stopped') except KeyboardInterrupt: self.log.info('Keyboard interrupt, request a global shutdown.') self.server.shutdown() except Exception: self.log.exception('The webserver exploded.') @botcmd(template='webstatus') def webstatus(self, msg, args): """ Gives a quick status of what is mapped in the internal webserver """ return {'rules': (((rule.rule, rule.endpoint) for rule in flask_app.url_map._rules))} @webhook def echo(self, incoming_request): """ A simple test webhook """ self.log.debug("Your incoming request is :" + str(incoming_request)) return str(incoming_request) @botcmd(split_args_with=' ') def webhook_test(self, _, args): """ Test your webhooks from within err. The syntax is : !webhook test [relative_url] [post content] It triggers the notification and generate also a little test report. """ url = args[0] content = ' '.join(args[1:]) # try to guess the content-type of what has been passed try: # try if it is plain json loads(content) contenttype = 'application/json' except ValueError: # try if it is a form splitted = content.split('=') # noinspection PyBroadException try: payload = '='.join(splitted[1:]) loads(unquote(payload)) contenttype = 'application/x-www-form-urlencoded' except Exception as _: contenttype = 'text/plain' # dunno what it is self.log.debug('Detected your post as : %s.', contenttype) response = self.test_app.post(url, params=content, content_type=contenttype) return TEST_REPORT % (url, contenttype, response.status_code) @botcmd(admin_only=True) def generate_certificate(self, _, args): """ Generate a self-signed SSL certificate for the Webserver """ yield ('Generating a new private key and certificate. This could take a ' 'while if your system is slow or low on entropy') key_path = os.sep.join((self.bot_config.BOT_DATA_DIR, "webserver_key.pem")) cert_path = os.sep.join((self.bot_config.BOT_DATA_DIR, "webserver_certificate.pem")) make_ssl_certificate(key_path=key_path, cert_path=cert_path) yield f'Certificate successfully generated and saved in {self.bot_config.BOT_DATA_DIR}.' suggested_config = self.config suggested_config['SSL']['enabled'] = True suggested_config['SSL']['host'] = suggested_config['HOST'] suggested_config['SSL']['port'] = suggested_config['PORT'] + 1 suggested_config['SSL']['key'] = key_path suggested_config['SSL']['certificate'] = cert_path yield 'To enable SSL with this certificate, the following config is recommended:' yield f'{suggested_config!r}'
class ConfigDashboard(threading.Thread): def __init__( self, running_args: argparse.Namespace, immutable_args: Iterable[str], dashboard_host: str, dashboard_port: int, **kwargs, ): threading.Thread.__init__(self) self.app = dash.Dash(__name__, url_base_pathname='/radiotracking-config/', meta_tags=[{ "name": "viewport", "content": "width=device-width, initial-scale=1" }]) config_columns = html.Div(children=[], style={ "columns": "2 359px", "padding": "20pt" }) config_tab = dcc.Tab(label="tRackIT Configuration", children=[config_columns]) config_columns.children.append( html.Div( "Reconfiguration requires restarting of pyradiotracking. Please keep in mind, that a broken configuration might lead to failing starts." )) self.running_args = running_args self.immutable_args = immutable_args self.config_states: List[State] = [] for group in Runner.parser._action_groups: # skip untitled groups if not isinstance(group.title, str): continue # skip groups not used in the config file if len(group._group_actions) == 0: continue group_div = html.Div(children=[], style={"break-inside": "avoid-column"}) config_columns.children.append(group_div) group_div.children.append(html.H3(f"[{group.title}]")) # iterate actions and extract values for action in group._group_actions: if action.dest not in vars(running_args): continue value = vars(running_args)[action.dest] group_div.children.append( html.P(children=[ html.B(action.dest), f" - {action.help}", html.Br(), dcc.Input(id=action.dest, value=repr(value)), ])) if action.type == int or isinstance(action, argparse._CountAction): if not isinstance(value, list): group_div.children[-1].children[-1].type = "number" group_div.children[-1].children[-1].step = 1 elif action.type == float: if not isinstance(value, list): group_div.children[-1].children[-1].type = "number" elif action.type == str: group_div.children[-1].children[-1].type = "text" if isinstance(value, list): group_div.children[-1].children[-1].value = repr(value) else: group_div.children[-1].children[-1].value = value elif isinstance(action, argparse._StoreTrueAction): group_div.children[-1].children[-1] = dcc.Checklist( id=action.dest, options=[ { "value": action.dest, "disabled": action.dest in self.immutable_args }, ], value=[action.dest] if value else [], ) if action.dest in self.immutable_args: group_div.children[-1].children[-1].disabled = True self.config_states.append(State(action.dest, "value")) config_columns.children.append(html.Button('Save', id="submit-config")) self.app.callback(Output('config-msg', 'children'), [ Input("submit-config", "n_clicks"), ], self.config_states)(self.submit_config) config_columns.children.append( html.Button('Restart', id="submit-restart")) self.app.callback(Output('submit-restart', 'children'), [ Input("submit-restart", "n_clicks"), ])(self.submit_restart) config_columns.children.append( html.H4("", id="config-msg", style={ "text-align": "center", "padding": "10px" })) tabs = dcc.Tabs(children=[]) tabs.children.append(config_tab) self.app.layout = html.Div([tabs]) self.app.layout.style = {"font-family": "sans-serif"} self.server = ThreadedWSGIServer(dashboard_host, dashboard_port + 1, self.app.server) self.calibrations: Dict[float, Dict[str, float]] = {} def _update_values(self): for el in self.app.layout._traverse(): if getattr(el, "id", None): if not el.id in self.running_args: continue value = vars(self.running_args)[el.id] if isinstance(value, bool): el.value = [el.id] if value else [] else: el.value = repr(value) def submit_config(self, clicks, *form_args): msg = html.Div(children=[]) args = self.running_args if not clicks: return msg for dest, value in zip( [state.component_id for state in self.config_states], form_args): # find corresponding action for action in Runner.parser._actions: # ignore immutable args if action.dest in self.immutable_args: continue if action.dest == dest: # boolean values are returned as lists, check if id is set if isinstance(value, list): args.__dict__[dest] = (dest in value) continue try: args.__dict__[dest] = literal_eval(value) except (ValueError, SyntaxError): args.__dict__[dest] = value except Exception as e: msg.children.append( html. P(f"Error: value for '{dest}' invalid ({repr(e)})." )) return msg # write config to actual location try: Runner.parser.write_config(args, open(self.running_args.config, "w")) except Exception as e: msg.children.append(html.P(str(e))) return msg self._update_values() msg.children.append( html.P(f"Config successfully written to '{args.config}'.")) return msg def submit_restart(self, clicks): if not clicks: return "Restart" # this is oddly specific and should be generalized os.system("systemctl restart radiotracking") return "Restarting..." def run(self): self.server.serve_forever() def stop(self): self.server.shutdown()
class Dashboard(AbstractConsumer, threading.Thread): def __init__( self, device: List[str], calibrate: bool, calibration: List[float], dashboard_host: str, dashboard_port: int, dashboard_signals: int, signal_min_duration_ms: int, signal_max_duration_ms: int, signal_threshold_dbw: float, snr_threshold_db: float, sample_rate: int, center_freq: int, signal_threshold_dbw_max: float = -20, snr_threshold_db_max: float = 50, **kwargs, ): threading.Thread.__init__(self) self.device = device self.calibrate = calibrate self.calibration = calibration self.signal_queue: Deque[Signal] = collections.deque( maxlen=dashboard_signals) self.matched_queue: Deque[MatchingSignal] = collections.deque( maxlen=dashboard_signals) # compute boundaries for sliders and initialize filters frequency_min = center_freq - sample_rate / 2 frequency_max = center_freq + sample_rate / 2 self.app = dash.Dash(__name__, url_base_pathname='/radiotracking/', meta_tags=[{ "name": "viewport", "content": "width=device-width, initial-scale=1" }]) graph_columns = html.Div(children=[], style={"columns": "2 359px"}) graph_columns.children.append( dcc.Graph(id="signal-noise", style={"break-inside": "avoid-column"})) self.app.callback(Output("signal-noise", "figure"), [ Input("update", "n_intervals"), Input("power-slider", "value"), Input("snr-slider", "value"), Input("frequency-slider", "value"), Input("duration-slider", "value"), ])(self.update_signal_noise) graph_columns.children.append( dcc.Graph(id="frequency-histogram", style={"break-inside": "avoid-column"})) self.app.callback(Output("frequency-histogram", "figure"), [ Input("update", "n_intervals"), Input("power-slider", "value"), Input("snr-slider", "value"), Input("frequency-slider", "value"), Input("duration-slider", "value"), ])(self.update_frequency_histogram) graph_columns.children.append( dcc.Graph(id="signal-match", style={"break-inside": "avoid-column"})) self.app.callback(Output("signal-match", "figure"), [ Input("update", "n_intervals"), ])(self.update_signal_match) graph_columns.children.append( dcc.Graph(id="signal-variance", style={"break-inside": "avoid-column"})) self.app.callback(Output("signal-variance", "figure"), [ Input("update", "n_intervals"), Input("power-slider", "value"), Input("snr-slider", "value"), Input("frequency-slider", "value"), Input("duration-slider", "value"), ])(self.update_signal_variance) graph_tab = dcc.Tab(label="tRackIT Signals", children=[]) graph_tab.children.append( html.H4("Running in calibration mode.", hidden=not calibrate, id="calibration-banner", style={ "text-align": "center", "width": "100%", "background-color": "#ffcccb", "padding": "20px", })) self.app.callback(Output("calibration-banner", "hidden"), [ Input("update", "n_intervals"), ])(self.update_calibration_banner) graph_tab.children.append(dcc.Interval(id="update", interval=1000)) self.app.callback(Output("update", "interval"), [Input("interval-slider", "value")])( self.update_interval) graph_tab.children.append(html.Div([ dcc.Graph(id="signal-time"), ])) self.app.callback(Output("signal-time", "figure"), [ Input("update", "n_intervals"), Input("power-slider", "value"), Input("snr-slider", "value"), Input("frequency-slider", "value"), Input("duration-slider", "value"), ])(self.update_signal_time) graph_columns.children.append( html.Div(children=[], id="calibration_output")) self.app.callback(Output("calibration_output", "children"), [ Input("update", "n_intervals"), ])(self.update_calibration) graph_columns.children.append( html.Div( id="settings", style={"break-inside": "avoid-column"}, children=[ html.H2("Vizualization Filters"), html.H3("Signal Power"), dcc.RangeSlider( id="power-slider", min=signal_threshold_dbw, max=signal_threshold_dbw_max, step=0.1, value=[signal_threshold_dbw, signal_threshold_dbw_max], marks={ int(signal_threshold_dbw): f"{signal_threshold_dbw} dBW", int(signal_threshold_dbw_max): f"{signal_threshold_dbw_max} dBW", }, ), html.H3("SNR"), dcc.RangeSlider( id="snr-slider", min=snr_threshold_db, max=snr_threshold_db_max, step=0.1, value=[snr_threshold_db, snr_threshold_db_max], marks={ int(snr_threshold_db): f"{snr_threshold_db} dBW", int(snr_threshold_db_max): f"{snr_threshold_db_max} dBW", }, ), html.H3("Frequency Range"), dcc.RangeSlider( id="frequency-slider", min=frequency_min, max=frequency_max, step=1, marks={ int(frequency_min): f"{frequency_min/1000/1000:.2f} MHz", int(center_freq): f"{center_freq/1000/1000:.2f} MHz", int(frequency_max): f"{frequency_max/1000/1000:.2f} MHz", }, value=[frequency_min, frequency_max], allowCross=False, ), html.H3("Signal Duration"), dcc.RangeSlider( id="duration-slider", min=signal_min_duration_ms, max=signal_max_duration_ms, step=0.1, marks={ int(signal_min_duration_ms): f"{signal_min_duration_ms} ms", int(signal_max_duration_ms): f"{signal_max_duration_ms} ms", }, value=[signal_min_duration_ms, signal_max_duration_ms], allowCross=False, ), html.H2("Dashboard Update Interval"), dcc.Slider( id="interval-slider", min=0.1, max=10, step=0.1, value=1.0, marks={ 0.1: "0.1 s", 1: "1 s", 5: "5 s", 10: "10 s", }, ), ])) graph_tab.children.append(graph_columns) tabs = dcc.Tabs(children=[]) tabs.children.append(graph_tab) self.app.layout = html.Div([tabs]) self.app.layout.style = {"font-family": "sans-serif"} self.server = ThreadedWSGIServer(dashboard_host, dashboard_port, self.app.server) self.calibrations: Dict[float, Dict[str, float]] = {} def add(self, signal: AbstractSignal): if isinstance(signal, Signal): self.signal_queue.append(signal) # create / update calibration dict calibrations[freq][device] = max(sig.avg) if signal.frequency not in self.calibrations: self.calibrations[signal.frequency] = {} # if freq has no avg for device, set it, else update with max of old and new if signal.device not in self.calibrations[signal.frequency]: self.calibrations[signal.frequency][signal.device] = signal.avg else: self.calibrations[signal.frequency][signal.device] = max( self.calibrations[signal.frequency][signal.device], signal.avg) elif isinstance(signal, MatchingSignal): self.matched_queue.append(signal) def update_calibration(self, n): header = html.Tr(children=[ html.Th("Frequency (MHz)"), html.Th("Max (dBW)"), ], style={"text-align": "left"}) settings_row = html.Tr(children=[ html.Td("[current settings]"), html.Td(""), ]) for device, calibration in zip(self.device, self.calibration): header.children.append(html.Th(f"SDR {device} (dB)")) settings_row.children.append(html.Td(f"{calibration:.2f}")) table = html.Table(children=[header, settings_row], style={ "width": "100%", "text-align": "left" }) for freq, avgs in sorted(self.calibrations.items(), key=lambda item: max(item[1])): ordered_avgs = [ avgs[d] + old if d in avgs else float("-inf") for d, old in zip(self.device, self.calibration) ] freq_max = max(ordered_avgs) row = html.Tr(children=[ html.Td(f"{freq/1000/1000:.3f}"), html.Td(f"{freq_max:.2f}") ]) for avg in ordered_avgs: row.children.append(html.Td(f"{avg - freq_max:.2f}")) table.children.append(row) return html.Div([ html.H2("Calibration Table"), table, ], style={"break-inside": "avoid-column"}) def update_calibration_banner(self, n): return not self.calibrate def update_interval(self, interval): return interval * 1000 def select_sigs(self, power: List[float], snr: List[float], freq: List[float], duration: List[float]): return [ sig for sig in self.signal_queue if sig.avg > power[0] and sig.avg < power[1] and sig.snr > snr[0] and sig.snr < snr[1] and sig.frequency > freq[0] and sig.frequency < freq[1] and sig.duration.total_seconds() * 1000 > duration[0] and sig.duration.total_seconds() * 1000 < duration[1] ] def update_signal_time(self, n, power, snr, freq, duration): traces = [] sigs = self.select_sigs(power, snr, freq, duration) for trace_sdr, sdr_sigs in group(sigs, "device"): trace = go.Scatter( x=[sig.ts for sig in sdr_sigs], y=[sig.avg for sig in sdr_sigs], name=trace_sdr, mode="markers", marker=dict( size=[ sig.duration.total_seconds() * 1000 for sig in sdr_sigs ], opacity=0.5, color=SDR_COLORS[trace_sdr], ), ) traces.append(trace) return { "data": traces, "layout": { "xaxis": { "title": "Time", "range": (sigs[0].ts if sigs else None, datetime.datetime.utcnow()) }, "yaxis": { "title": "Signal Power (dBW)", "range": power }, "legend": { "title": "SDR Receiver" }, }, } def update_signal_noise(self, n, power, snr, freq, duration): traces = [] sigs = self.select_sigs(power, snr, freq, duration) for trace_sdr, sdr_sigs in group(sigs, "device"): trace = go.Scatter( x=[sig.snr for sig in sdr_sigs], y=[sig.avg for sig in sdr_sigs], name=trace_sdr, mode="markers", marker=dict( size=[ sig.duration.total_seconds() * 1000 for sig in sdr_sigs ], opacity=0.3, color=SDR_COLORS[trace_sdr], ), ) traces.append(trace) return { "data": traces, "layout": { "title": "Signal to Noise", "xaxis": { "title": "SNR (dB)" }, "yaxis": { "title": "Signal Power (dBW)", "range": power }, "legend": { "title": "SDR Receiver" }, }, } def update_signal_variance(self, n, power, snr, freq, duration): traces = [] sigs = self.select_sigs(power, snr, freq, duration) for trace_sdr, sdr_sigs in group(sigs, "device"): trace = go.Scatter( x=[sig.std for sig in sdr_sigs], y=[sig.avg for sig in sdr_sigs], name=trace_sdr, mode="markers", marker=dict( size=[ sig.duration.total_seconds() * 1000 for sig in sdr_sigs ], opacity=0.3, color=SDR_COLORS[trace_sdr], ), ) traces.append(trace) return { "data": traces, "layout": { "title": "Signal Variance", "xaxis": { "title": "Standard Deviation (dB)" }, "yaxis": { "title": "Signal Power (dBW)", "range": power }, "legend": { "title": "SDR Receiver" }, }, } def update_frequency_histogram(self, n, power, snr, freq, duration): traces = [] sigs = self.select_sigs(power, snr, freq, duration) for trace_sdr, sdr_sigs in group(sigs, "device"): trace = go.Scatter( x=[sig.frequency for sig in sdr_sigs], y=[sig.avg for sig in sdr_sigs], name=trace_sdr, mode="markers", marker=dict( size=[ sig.duration.total_seconds() * 1000 for sig in sdr_sigs ], opacity=0.3, color=SDR_COLORS[trace_sdr], ), ) traces.append(trace) return { "data": traces, "layout": { "title": "Frequency Usage", "xaxis": { "title": "Frequency (MHz)", "range": freq }, "yaxis": { "title": "Signal Power (dBW)", "range": power }, "legend_title_text": "SDR Receiver", }, } def update_signal_match(self, n): traces = [] completed_signals = [ msig for msig in self.matched_queue if len(msig._sigs) == 4 ] trace = go.Scatter( x=[ msig._sigs["0"].avg - msig._sigs["2"].avg for msig in completed_signals ], y=[ msig._sigs["1"].avg - msig._sigs["3"].avg for msig in completed_signals ], mode="markers", marker=dict( color=[msig.ts.timestamp() for msig in completed_signals], colorscale='Cividis_r', opacity=0.5, )) traces.append(trace) return { "data": traces, "layout": { "title": "Matched Frequencies", "xaxis": { "title": "Horizontal Difference", "range": [-50, 50], }, "yaxis": { "title": "Vertical Difference", "range": [-50, 50], }, }, } def run(self): self.server.serve_forever() def stop(self): self.server.shutdown()
class Webserver(BotPlugin): def __init__(self, *args, **kwargs): self.server = None self.server_thread = None self.ssl_context = None self.test_app = TestApp(flask_app) super().__init__(*args, **kwargs) def get_configuration_template(self): return { "HOST": "0.0.0.0", "PORT": 3141, "SSL": { "enabled": False, "host": "0.0.0.0", "port": 3142, "certificate": "", "key": "", }, } def check_configuration(self, configuration): # it is a pain, just assume a default config if SSL is absent or set to None if configuration.get("SSL", None) is None: configuration["SSL"] = { "enabled": False, "host": "0.0.0.0", "port": 3142, "certificate": "", "key": "", } super().check_configuration(configuration) def activate(self): if not self.config: self.log.info("Webserver is not configured. Forbid activation") return if self.server_thread and self.server_thread.is_alive(): raise Exception( "Invalid state, you should not have a webserver already running." ) self.server_thread = Thread(target=self.run_server, name="Webserver Thread") self.server_thread.start() self.log.debug("Webserver started.") super().activate() def deactivate(self): if self.server is not None: self.log.info("Shutting down the internal webserver.") self.server.shutdown() self.log.info("Waiting for the webserver thread to quit.") self.server_thread.join() self.log.info("Webserver shut down correctly.") super().deactivate() def run_server(self): try: host = self.config["HOST"] port = self.config["PORT"] ssl = self.config["SSL"] self.log.info("Starting the webserver on %s:%i", host, port) ssl_context = (ssl["certificate"], ssl["key"]) if ssl["enabled"] else None self.server = ThreadedWSGIServer( host, ssl["port"] if ssl_context else port, flask_app, ssl_context=ssl_context, ) self.server.serve_forever() self.log.debug("Webserver stopped") except KeyboardInterrupt: self.log.info("Keyboard interrupt, request a global shutdown.") self.server.shutdown() except Exception: self.log.exception("The webserver exploded.") @botcmd(template="webstatus") def webstatus(self, msg, args): """ Gives a quick status of what is mapped in the internal webserver """ return { "rules": (((rule.rule, rule.endpoint) for rule in flask_app.url_map._rules)) } @webhook def echo(self, incoming_request): """ A simple test webhook """ self.log.debug("Your incoming request is: %s", incoming_request) return str(incoming_request) @botcmd(split_args_with=" ") def webhook_test(self, _, args): """ Test your webhooks from within err. The syntax is : !webhook test [relative_url] [post content] It triggers the notification and generate also a little test report. """ url = args[0] content = " ".join(args[1:]) # try to guess the content-type of what has been passed try: # try if it is plain json loads(content) contenttype = "application/json" except ValueError: # try if it is a form splitted = content.split("=") # noinspection PyBroadException try: payload = "=".join(splitted[1:]) loads(unquote(payload)) contenttype = "application/x-www-form-urlencoded" except Exception as _: contenttype = "text/plain" # dunno what it is self.log.debug("Detected your post as : %s.", contenttype) response = self.test_app.post(url, params=content, content_type=contenttype) return TEST_REPORT % (url, contenttype, response.status_code) @botcmd(admin_only=True) def generate_certificate(self, _, args): """ Generate a self-signed SSL certificate for the Webserver """ yield ( "Generating a new private key and certificate. This could take a " "while if your system is slow or low on entropy") key_path = os.sep.join( (self.bot_config.BOT_DATA_DIR, "webserver_key.pem")) cert_path = os.sep.join( (self.bot_config.BOT_DATA_DIR, "webserver_certificate.pem")) make_ssl_certificate(key_path=key_path, cert_path=cert_path) yield f"Certificate successfully generated and saved in {self.bot_config.BOT_DATA_DIR}." suggested_config = self.config suggested_config["SSL"]["enabled"] = True suggested_config["SSL"]["host"] = suggested_config["HOST"] suggested_config["SSL"]["port"] = suggested_config["PORT"] + 1 suggested_config["SSL"]["key"] = key_path suggested_config["SSL"]["certificate"] = cert_path yield "To enable SSL with this certificate, the following config is recommended:" yield f"{suggested_config!r}"