def restart_threads(self, *args): logger.info("Restart threads") if self.raw_report_processor.is_alive(): self.raw_report_processor.stop() self.raw_report_processor.join() self.raw_report_processor = ReportsManager(REPORTS_QUEUE) self.raw_report_processor.start()
def run(self): def signal_handler(*args): logger.info('Received SIGTERM, shutting down.') logger.info("Stopping threads, please wait...") self.__stop_all_threads() log_path = CONST_FARADAY_HOME_PATH / 'logs' / 'access-logging.log' site = twisted.web.server.Site(self.__root_resource, logPath=log_path, logFormatter=proxiedLogFormatter) site.displayTracebacks = False self.__listen_func = reactor.listenTCP try: self.install_signal() # start threads and processes self.raw_report_processor = ReportsManager( REPORTS_QUEUE, name="ReportsManager-Thread", daemon=True) self.raw_report_processor.start() self.ping_home_thread = PingHomeThread() self.ping_home_thread.start() # web and static content self.__listen_func(self.__listen_port, site, interface=self.__bind_address) # websockets try: listenWS(self.__build_websockets_resource(), interface=self.__bind_address) except error.CannotListenError: logger.warn( 'Could not start websockets, address already open. This is ok is you wan to run multiple instances.' ) except Exception as ex: logger.warn(f'Could not start websocket, error: {ex}') logger.info('Faraday Server is ready') reactor.addSystemEventTrigger('before', 'shutdown', signal_handler) reactor.run() except error.CannotListenError as e: logger.error(e) self.__stop_all_threads() sys.exit(1) except Exception as e: logger.exception( 'Something went wrong when trying to setup the Web UI') logger.exception(e) self.__stop_all_threads() sys.exit(1)
def test_file_upload(self, test_client, session, csrf_token, logged_user): ws = WorkspaceFactory.create(name="abc") session.add(ws) session.commit() path = TEST_DATA_PATH / 'nmap_plugin_with_api.xml' with path.open('rb') as report: file_contents = report.read() data = { 'file': (BytesIO(file_contents), 'nmap_report.xml'), 'csrf_token': csrf_token } res = test_client.post( self.check_url(f'/v2/ws/{ws.name}/upload_report'), data=data, use_json_data=False) assert res.status_code == 200 assert len(REPORTS_QUEUE.queue) == 1 queue_elem = REPORTS_QUEUE.get_nowait() assert queue_elem[0] == ws.name assert queue_elem[3].lower() == "nmap" assert queue_elem[4] == logged_user.id # I'm testing a method which lost referene of workspace and logged_user within the test ws_id = ws.id logged_user_id = logged_user.id from faraday.server.threads.reports_processor import ReportsManager false_thread = ReportsManager(None) false_thread.process_report(queue_elem[0], queue_elem[1], queue_elem[2], queue_elem[3], queue_elem[4]) command = Command.query.filter(Command.workspace_id == ws_id).one() assert command assert command.creator_id == logged_user_id assert command.id == res.json["command_id"] assert command.end_date host = Host.query.filter(Host.workspace_id == ws_id).first() assert host assert host.creator_id == logged_user_id service = Service.query.filter(Service.workspace_id == ws_id).first() assert service assert service.creator_id == logged_user_id
def test_file_upload(self, test_client, session, csrf_token, logged_user): ws = WorkspaceFactory.create(name="abc") session.add(ws) session.commit() path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data', 'nmap_plugin_with_api.xml') with open(path, 'rb') as report: file_contents = report.read() data = { 'file': (BytesIO(file_contents), 'nmap_report.xml'), 'csrf_token': csrf_token } res = test_client.post( '/v2/ws/{ws_name}/upload_report'.format(ws_name=ws.name), data=data, use_json_data=False) assert res.status_code == 200 assert len(REPORTS_QUEUE.queue) == 1 queue_elem = REPORTS_QUEUE.queue[0] assert queue_elem[0] == ws.name assert queue_elem[2].id == logged_user.id # I'm testing a method which lost referene of workspace and logged_user within the test ws_id = ws.id logged_user_id = logged_user.id from faraday.server.threads.reports_processor import ReportsManager false_thread = ReportsManager(None) false_thread.process_report(queue_elem[0], queue_elem[1], queue_elem[2]) command = Command.query.filter(Command.workspace_id == ws_id).one() assert command assert command.creator_id == logged_user_id host = Host.query.filter(Host.workspace_id == ws_id).first() assert host assert host.creator_id == logged_user_id service = Service.query.filter(Service.workspace_id == ws_id).first() assert service assert service.creator_id == logged_user_id
class WebServer: UI_URL_PATH = b'_ui' API_URL_PATH = b'_api' WEB_UI_LOCAL_PATH = faraday.server.config.FARADAY_BASE / 'server/www' def __init__(self): self.__ssl_enabled = faraday.server.config.ssl.enabled logger.info( 'Starting web server at %s://%s:%s/', 'https' if self.__ssl_enabled else 'http', faraday.server.config.faraday_server.bind_address, faraday.server.config.ssl.port if self.__ssl_enabled else faraday.server.config.faraday_server.port) self.__websocket_ssl_enabled = faraday.server.config.websocket_ssl.enabled self.__websocket_port = faraday.server.config.faraday_server.websocket_port or 9000 self.__config_server() self.__build_server_tree() def __config_server(self): self.__bind_address = faraday.server.config.faraday_server.bind_address if self.__ssl_enabled: self.__listen_port = int(faraday.server.config.ssl.port) else: self.__listen_port = int(faraday.server.config.faraday_server.port) def __load_ssl_certs(self): certs = (faraday.server.config.ssl.keyfile, faraday.server.config.ssl.certificate) if not all(certs): logger.critical( "HTTPS request but SSL certificates are not configured") sys.exit(1) # Abort web-server startup return ssl.DefaultOpenSSLContextFactory(*certs) def __build_server_tree(self): self.__root_resource = self.__build_web_resource() self.__root_resource.putChild(WebServer.UI_URL_PATH, self.__build_web_redirect()) self.__root_resource.putChild(WebServer.API_URL_PATH, self.__build_api_resource()) def __build_web_redirect(self): return FaradayRedirectResource(b'/') def __build_web_resource(self): return FileWithoutDirectoryListing(WebServer.WEB_UI_LOCAL_PATH) def __build_api_resource(self): return FaradayWSGIResource(reactor, reactor.getThreadPool(), app) def __build_websockets_resource(self): websocket_port = int( faraday.server.config.faraday_server.websocket_port) url = f'{self.__bind_address}:{websocket_port}/websockets' if self.__websocket_ssl_enabled: url = 'wss://' + url else: url = 'ws://' + url # logger.info(u"Websocket listening at {url}".format(url=url)) logger.info( 'Starting websocket server at port {0} with bind address {1}. ' 'SSL {2}'.format(self.__websocket_port, self.__bind_address, self.__ssl_enabled)) factory = WorkspaceServerFactory(url=url) factory.protocol = BroadcastServerProtocol return factory def __stop_all_threads(self): if self.raw_report_processor.is_alive(): self.raw_report_processor.stop() self.ping_home_thread.stop() def install_signal(self): for sig in (SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM): signal(sig, SIG_DFL) def run(self): def signal_handler(*args): logger.info('Received SIGTERM, shutting down.') logger.info("Stopping threads, please wait...") self.__stop_all_threads() log_path = CONST_FARADAY_HOME_PATH / 'logs' / 'access-logging.log' site = twisted.web.server.Site(self.__root_resource, logPath=log_path, logFormatter=proxiedLogFormatter) site.displayTracebacks = False if self.__ssl_enabled: ssl_context = self.__load_ssl_certs() self.__listen_func = functools.partial(reactor.listenSSL, contextFactory=ssl_context) else: self.__listen_func = reactor.listenTCP try: self.install_signal() # start threads and processes self.raw_report_processor = ReportsManager( REPORTS_QUEUE, name="ReportsManager-Thread", daemon=True) self.raw_report_processor.start() self.ping_home_thread = PingHomeThread() self.ping_home_thread.start() # web and static content self.__listen_func(self.__listen_port, site, interface=self.__bind_address) # websockets if faraday.server.config.websocket_ssl.enabled: try: contextFactory = ssl.DefaultOpenSSLContextFactory( faraday.server.config.websocket_ssl.keyfile.strip( '\''), faraday.server.config.websocket_ssl.certificate.strip( '\'')) listenWS(self.__build_websockets_resource(), interface=self.__bind_address, contextFactory=contextFactory) except SSLError as e: logger.error( 'Could not start websockets due to a SSL Config error. Some web functionality will not be available' ) except error.CannotListenError: logger.warn( 'Could not start websockets, address already open. This is ok is you wan to run multiple instances.' ) except Exception as ex: logger.warn(f'Could not start websocket, error: {ex}') else: try: listenWS(self.__build_websockets_resource(), interface=self.__bind_address) except error.CannotListenError: logger.warn( 'Could not start websockets, address already open. This is ok is you wan to run multiple instances.' ) except Exception as ex: logger.warn(f'Could not start websocket, error: {ex}') logger.info('Faraday Server is ready') reactor.addSystemEventTrigger('before', 'shutdown', signal_handler) reactor.run() except error.CannotListenError as e: logger.error(e) self.__stop_all_threads() sys.exit(1) except Exception as e: logger.exception( 'Something went wrong when trying to setup the Web UI') logger.exception(e) self.__stop_all_threads() sys.exit(1)
def run(self): def signal_handler(*args): logger.info('Received SIGTERM, shutting down.') logger.info("Stopping threads, please wait...") self.__stop_all_threads() log_path = CONST_FARADAY_HOME_PATH / 'logs' / 'access-logging.log' site = twisted.web.server.Site(self.__root_resource, logPath=log_path, logFormatter=proxiedLogFormatter) site.displayTracebacks = False if self.__ssl_enabled: ssl_context = self.__load_ssl_certs() self.__listen_func = functools.partial(reactor.listenSSL, contextFactory=ssl_context) else: self.__listen_func = reactor.listenTCP try: self.install_signal() # start threads and processes self.raw_report_processor = ReportsManager( REPORTS_QUEUE, name="ReportsManager-Thread", daemon=True) self.raw_report_processor.start() self.ping_home_thread = PingHomeThread() self.ping_home_thread.start() # web and static content self.__listen_func(self.__listen_port, site, interface=self.__bind_address) # websockets if faraday.server.config.websocket_ssl.enabled: try: contextFactory = ssl.DefaultOpenSSLContextFactory( faraday.server.config.websocket_ssl.keyfile.strip( '\''), faraday.server.config.websocket_ssl.certificate.strip( '\'')) listenWS(self.__build_websockets_resource(), interface=self.__bind_address, contextFactory=contextFactory) except SSLError as e: logger.error( 'Could not start websockets due to a SSL Config error. Some web functionality will not be available' ) except error.CannotListenError: logger.warn( 'Could not start websockets, address already open. This is ok is you wan to run multiple instances.' ) except Exception as ex: logger.warn(f'Could not start websocket, error: {ex}') else: try: listenWS(self.__build_websockets_resource(), interface=self.__bind_address) except error.CannotListenError: logger.warn( 'Could not start websockets, address already open. This is ok is you wan to run multiple instances.' ) except Exception as ex: logger.warn(f'Could not start websocket, error: {ex}') logger.info('Faraday Server is ready') reactor.addSystemEventTrigger('before', 'shutdown', signal_handler) reactor.run() except error.CannotListenError as e: logger.error(e) self.__stop_all_threads() sys.exit(1) except Exception as e: logger.exception( 'Something went wrong when trying to setup the Web UI') logger.exception(e) self.__stop_all_threads() sys.exit(1)
class WebServer: API_URL_PATH = b'_api' WEB_UI_LOCAL_PATH = faraday.server.config.FARADAY_BASE / 'server/www' # Threads raw_report_processor = None ping_home_thread = None def __init__(self): logger.info('Starting web server at http://' f'{server_config.bind_address}:' f'{server_config.port}/') self.__build_server_tree() def __build_server_tree(self): self.root_resource = self.__build_web_resource() self.root_resource.putChild(WebServer.API_URL_PATH, self.__build_api_resource()) def __build_web_resource(self): return FileWithoutDirectoryListing(WebServer.WEB_UI_LOCAL_PATH) def __build_api_resource(self): return FaradayWSGIResource(reactor, reactor.getThreadPool(), get_app()) def __build_websockets_resource(self): url = f'ws://{server_config.bind_address}:{server_config.websocket_port}/websockets' logger.info( f'Starting websocket server at port ' f'{server_config.websocket_port} with bind address {server_config.bind_address}.' ) factory = WorkspaceServerFactory(url=url) factory.protocol = BroadcastServerProtocol return factory def install_signal(self): for sig in (SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM): signal(sig, SIG_DFL) def stop_threads(self): logger.info("Stopping threads...") if self.raw_report_processor.is_alive(): self.raw_report_processor.stop() if self.ping_home_thread.is_alive(): self.ping_home_thread.stop() def restart_threads(self, *args): logger.info("Restart threads") if self.raw_report_processor.is_alive(): self.raw_report_processor.stop() self.raw_report_processor.join() self.raw_report_processor = ReportsManager(REPORTS_QUEUE) self.raw_report_processor.start() def start_threads(self): self.raw_report_processor = ReportsManager(REPORTS_QUEUE) self.raw_report_processor.start() self.ping_home_thread = PingHomeThread() self.ping_home_thread.start() def run(self): def signal_handler(*args): logger.info('Received SIGTERM, shutting down.') logger.info("Stopping threads, please wait...") self.stop_threads() log_path = CONST_FARADAY_HOME_PATH / 'logs' / 'access-logging.log' site = FaradaySite(self.root_resource, logPath=log_path, logFormatter=proxiedLogFormatter) site.displayTracebacks = False try: self.install_signal() # start threads and processes self.start_threads() # web and static content reactor.listenTCP(server_config.port, site, interface=server_config.bind_address) num_threads = multiprocessing.cpu_count() * 2 logger.info(f'Starting webserver with {num_threads} threads.') reactor.suggestThreadPoolSize(num_threads) # websockets try: listenWS(self.__build_websockets_resource(), interface=server_config.bind_address) except error.CannotListenError: logger.warn( 'Could not start websockets, address already open. This is ok is you wan to run multiple instances.' ) except Exception as ex: logger.warn(f'Could not start websocket, error: {ex}') logger.info('Faraday Server is ready') reactor.addSystemEventTrigger('before', 'shutdown', signal_handler) signal(SIGUSR1, self.restart_threads) reactor.run() except error.CannotListenError as e: logger.error(e) self.stop_threads() sys.exit(1) except Exception as e: logger.exception( 'Something went wrong when trying to setup the Web UI') logger.exception(e) self.stop_threads() sys.exit(1)
def start_threads(self): self.raw_report_processor = ReportsManager(REPORTS_QUEUE) self.raw_report_processor.start() self.ping_home_thread = PingHomeThread() self.ping_home_thread.start()