def restart_threads(self, *args):
logger.info("Restart threads")
if self.raw_report_processor.is_alive():
self.raw_report_processor.stop()
self.raw_report_processor.join()
self.raw_report_processor = ReportsManager(REPORTS_QUEUE)
self.raw_report_processor.start()
def run(self):
def signal_handler(*args):
logger.info('Received SIGTERM, shutting down.')
logger.info("Stopping threads, please wait...")
self.__stop_all_threads()
log_path = CONST_FARADAY_HOME_PATH / 'logs' / 'access-logging.log'
site = twisted.web.server.Site(self.__root_resource,
logPath=log_path,
logFormatter=proxiedLogFormatter)
site.displayTracebacks = False
self.__listen_func = reactor.listenTCP
try:
self.install_signal()
# start threads and processes
self.raw_report_processor = ReportsManager(
REPORTS_QUEUE, name="ReportsManager-Thread", daemon=True)
self.raw_report_processor.start()
self.ping_home_thread = PingHomeThread()
self.ping_home_thread.start()
# web and static content
self.__listen_func(self.__listen_port,
site,
interface=self.__bind_address)
# websockets
try:
listenWS(self.__build_websockets_resource(),
interface=self.__bind_address)
except error.CannotListenError:
logger.warn(
'Could not start websockets, address already open. This is ok is you wan to run multiple instances.'
)
except Exception as ex:
logger.warn(f'Could not start websocket, error: {ex}')
logger.info('Faraday Server is ready')
reactor.addSystemEventTrigger('before', 'shutdown', signal_handler)
reactor.run()
except error.CannotListenError as e:
logger.error(e)
self.__stop_all_threads()
sys.exit(1)
except Exception as e:
logger.exception(
'Something went wrong when trying to setup the Web UI')
logger.exception(e)
self.__stop_all_threads()
sys.exit(1)
def test_file_upload(self, test_client, session, csrf_token, logged_user):
ws = WorkspaceFactory.create(name="abc")
session.add(ws)
session.commit()
path = TEST_DATA_PATH / 'nmap_plugin_with_api.xml'
with path.open('rb') as report:
file_contents = report.read()
data = {
'file': (BytesIO(file_contents), 'nmap_report.xml'),
'csrf_token': csrf_token
}
res = test_client.post(
self.check_url(f'/v2/ws/{ws.name}/upload_report'),
data=data,
use_json_data=False)
assert res.status_code == 200
assert len(REPORTS_QUEUE.queue) == 1
queue_elem = REPORTS_QUEUE.get_nowait()
assert queue_elem[0] == ws.name
assert queue_elem[3].lower() == "nmap"
assert queue_elem[4] == logged_user.id
# I'm testing a method which lost referene of workspace and logged_user within the test
ws_id = ws.id
logged_user_id = logged_user.id
from faraday.server.threads.reports_processor import ReportsManager
false_thread = ReportsManager(None)
false_thread.process_report(queue_elem[0], queue_elem[1],
queue_elem[2], queue_elem[3],
queue_elem[4])
command = Command.query.filter(Command.workspace_id == ws_id).one()
assert command
assert command.creator_id == logged_user_id
assert command.id == res.json["command_id"]
assert command.end_date
host = Host.query.filter(Host.workspace_id == ws_id).first()
assert host
assert host.creator_id == logged_user_id
service = Service.query.filter(Service.workspace_id == ws_id).first()
assert service
assert service.creator_id == logged_user_id
def test_file_upload(self, test_client, session, csrf_token, logged_user):
ws = WorkspaceFactory.create(name="abc")
session.add(ws)
session.commit()
path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'data', 'nmap_plugin_with_api.xml')
with open(path, 'rb') as report:
file_contents = report.read()
data = {
'file': (BytesIO(file_contents), 'nmap_report.xml'),
'csrf_token': csrf_token
}
res = test_client.post(
'/v2/ws/{ws_name}/upload_report'.format(ws_name=ws.name),
data=data,
use_json_data=False)
assert res.status_code == 200
assert len(REPORTS_QUEUE.queue) == 1
queue_elem = REPORTS_QUEUE.queue[0]
assert queue_elem[0] == ws.name
assert queue_elem[2].id == logged_user.id
# I'm testing a method which lost referene of workspace and logged_user within the test
ws_id = ws.id
logged_user_id = logged_user.id
from faraday.server.threads.reports_processor import ReportsManager
false_thread = ReportsManager(None)
false_thread.process_report(queue_elem[0], queue_elem[1],
queue_elem[2])
command = Command.query.filter(Command.workspace_id == ws_id).one()
assert command
assert command.creator_id == logged_user_id
host = Host.query.filter(Host.workspace_id == ws_id).first()
assert host
assert host.creator_id == logged_user_id
service = Service.query.filter(Service.workspace_id == ws_id).first()
assert service
assert service.creator_id == logged_user_id
class WebServer:
UI_URL_PATH = b'_ui'
API_URL_PATH = b'_api'
WEB_UI_LOCAL_PATH = faraday.server.config.FARADAY_BASE / 'server/www'
def __init__(self):
self.__ssl_enabled = faraday.server.config.ssl.enabled
logger.info(
'Starting web server at %s://%s:%s/',
'https' if self.__ssl_enabled else 'http',
faraday.server.config.faraday_server.bind_address,
faraday.server.config.ssl.port if self.__ssl_enabled else
faraday.server.config.faraday_server.port)
self.__websocket_ssl_enabled = faraday.server.config.websocket_ssl.enabled
self.__websocket_port = faraday.server.config.faraday_server.websocket_port or 9000
self.__config_server()
self.__build_server_tree()
def __config_server(self):
self.__bind_address = faraday.server.config.faraday_server.bind_address
if self.__ssl_enabled:
self.__listen_port = int(faraday.server.config.ssl.port)
else:
self.__listen_port = int(faraday.server.config.faraday_server.port)
def __load_ssl_certs(self):
certs = (faraday.server.config.ssl.keyfile,
faraday.server.config.ssl.certificate)
if not all(certs):
logger.critical(
"HTTPS request but SSL certificates are not configured")
sys.exit(1) # Abort web-server startup
return ssl.DefaultOpenSSLContextFactory(*certs)
def __build_server_tree(self):
self.__root_resource = self.__build_web_resource()
self.__root_resource.putChild(WebServer.UI_URL_PATH,
self.__build_web_redirect())
self.__root_resource.putChild(WebServer.API_URL_PATH,
self.__build_api_resource())
def __build_web_redirect(self):
return FaradayRedirectResource(b'/')
def __build_web_resource(self):
return FileWithoutDirectoryListing(WebServer.WEB_UI_LOCAL_PATH)
def __build_api_resource(self):
return FaradayWSGIResource(reactor, reactor.getThreadPool(), app)
def __build_websockets_resource(self):
websocket_port = int(
faraday.server.config.faraday_server.websocket_port)
url = f'{self.__bind_address}:{websocket_port}/websockets'
if self.__websocket_ssl_enabled:
url = 'wss://' + url
else:
url = 'ws://' + url
# logger.info(u"Websocket listening at {url}".format(url=url))
logger.info(
'Starting websocket server at port {0} with bind address {1}. '
'SSL {2}'.format(self.__websocket_port, self.__bind_address,
self.__ssl_enabled))
factory = WorkspaceServerFactory(url=url)
factory.protocol = BroadcastServerProtocol
return factory
def __stop_all_threads(self):
if self.raw_report_processor.is_alive():
self.raw_report_processor.stop()
self.ping_home_thread.stop()
def install_signal(self):
for sig in (SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM):
signal(sig, SIG_DFL)
def run(self):
def signal_handler(*args):
logger.info('Received SIGTERM, shutting down.')
logger.info("Stopping threads, please wait...")
self.__stop_all_threads()
log_path = CONST_FARADAY_HOME_PATH / 'logs' / 'access-logging.log'
site = twisted.web.server.Site(self.__root_resource,
logPath=log_path,
logFormatter=proxiedLogFormatter)
site.displayTracebacks = False
if self.__ssl_enabled:
ssl_context = self.__load_ssl_certs()
self.__listen_func = functools.partial(reactor.listenSSL,
contextFactory=ssl_context)
else:
self.__listen_func = reactor.listenTCP
try:
self.install_signal()
# start threads and processes
self.raw_report_processor = ReportsManager(
REPORTS_QUEUE, name="ReportsManager-Thread", daemon=True)
self.raw_report_processor.start()
self.ping_home_thread = PingHomeThread()
self.ping_home_thread.start()
# web and static content
self.__listen_func(self.__listen_port,
site,
interface=self.__bind_address)
# websockets
if faraday.server.config.websocket_ssl.enabled:
try:
contextFactory = ssl.DefaultOpenSSLContextFactory(
faraday.server.config.websocket_ssl.keyfile.strip(
'\''),
faraday.server.config.websocket_ssl.certificate.strip(
'\''))
listenWS(self.__build_websockets_resource(),
interface=self.__bind_address,
contextFactory=contextFactory)
except SSLError as e:
logger.error(
'Could not start websockets due to a SSL Config error. Some web functionality will not be available'
)
except error.CannotListenError:
logger.warn(
'Could not start websockets, address already open. This is ok is you wan to run multiple instances.'
)
except Exception as ex:
logger.warn(f'Could not start websocket, error: {ex}')
else:
try:
listenWS(self.__build_websockets_resource(),
interface=self.__bind_address)
except error.CannotListenError:
logger.warn(
'Could not start websockets, address already open. This is ok is you wan to run multiple instances.'
)
except Exception as ex:
logger.warn(f'Could not start websocket, error: {ex}')
logger.info('Faraday Server is ready')
reactor.addSystemEventTrigger('before', 'shutdown', signal_handler)
reactor.run()
except error.CannotListenError as e:
logger.error(e)
self.__stop_all_threads()
sys.exit(1)
except Exception as e:
logger.exception(
'Something went wrong when trying to setup the Web UI')
logger.exception(e)
self.__stop_all_threads()
sys.exit(1)
def run(self):
def signal_handler(*args):
logger.info('Received SIGTERM, shutting down.')
logger.info("Stopping threads, please wait...")
self.__stop_all_threads()
log_path = CONST_FARADAY_HOME_PATH / 'logs' / 'access-logging.log'
site = twisted.web.server.Site(self.__root_resource,
logPath=log_path,
logFormatter=proxiedLogFormatter)
site.displayTracebacks = False
if self.__ssl_enabled:
ssl_context = self.__load_ssl_certs()
self.__listen_func = functools.partial(reactor.listenSSL,
contextFactory=ssl_context)
else:
self.__listen_func = reactor.listenTCP
try:
self.install_signal()
# start threads and processes
self.raw_report_processor = ReportsManager(
REPORTS_QUEUE, name="ReportsManager-Thread", daemon=True)
self.raw_report_processor.start()
self.ping_home_thread = PingHomeThread()
self.ping_home_thread.start()
# web and static content
self.__listen_func(self.__listen_port,
site,
interface=self.__bind_address)
# websockets
if faraday.server.config.websocket_ssl.enabled:
try:
contextFactory = ssl.DefaultOpenSSLContextFactory(
faraday.server.config.websocket_ssl.keyfile.strip(
'\''),
faraday.server.config.websocket_ssl.certificate.strip(
'\''))
listenWS(self.__build_websockets_resource(),
interface=self.__bind_address,
contextFactory=contextFactory)
except SSLError as e:
logger.error(
'Could not start websockets due to a SSL Config error. Some web functionality will not be available'
)
except error.CannotListenError:
logger.warn(
'Could not start websockets, address already open. This is ok is you wan to run multiple instances.'
)
except Exception as ex:
logger.warn(f'Could not start websocket, error: {ex}')
else:
try:
listenWS(self.__build_websockets_resource(),
interface=self.__bind_address)
except error.CannotListenError:
logger.warn(
'Could not start websockets, address already open. This is ok is you wan to run multiple instances.'
)
except Exception as ex:
logger.warn(f'Could not start websocket, error: {ex}')
logger.info('Faraday Server is ready')
reactor.addSystemEventTrigger('before', 'shutdown', signal_handler)
reactor.run()
except error.CannotListenError as e:
logger.error(e)
self.__stop_all_threads()
sys.exit(1)
except Exception as e:
logger.exception(
'Something went wrong when trying to setup the Web UI')
logger.exception(e)
self.__stop_all_threads()
sys.exit(1)
class WebServer:
API_URL_PATH = b'_api'
WEB_UI_LOCAL_PATH = faraday.server.config.FARADAY_BASE / 'server/www'
# Threads
raw_report_processor = None
ping_home_thread = None
def __init__(self):
logger.info('Starting web server at http://'
f'{server_config.bind_address}:'
f'{server_config.port}/')
self.__build_server_tree()
def __build_server_tree(self):
self.root_resource = self.__build_web_resource()
self.root_resource.putChild(WebServer.API_URL_PATH,
self.__build_api_resource())
def __build_web_resource(self):
return FileWithoutDirectoryListing(WebServer.WEB_UI_LOCAL_PATH)
def __build_api_resource(self):
return FaradayWSGIResource(reactor, reactor.getThreadPool(), get_app())
def __build_websockets_resource(self):
url = f'ws://{server_config.bind_address}:{server_config.websocket_port}/websockets'
logger.info(
f'Starting websocket server at port '
f'{server_config.websocket_port} with bind address {server_config.bind_address}.'
)
factory = WorkspaceServerFactory(url=url)
factory.protocol = BroadcastServerProtocol
return factory
def install_signal(self):
for sig in (SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM):
signal(sig, SIG_DFL)
def stop_threads(self):
logger.info("Stopping threads...")
if self.raw_report_processor.is_alive():
self.raw_report_processor.stop()
if self.ping_home_thread.is_alive():
self.ping_home_thread.stop()
def restart_threads(self, *args):
logger.info("Restart threads")
if self.raw_report_processor.is_alive():
self.raw_report_processor.stop()
self.raw_report_processor.join()
self.raw_report_processor = ReportsManager(REPORTS_QUEUE)
self.raw_report_processor.start()
def start_threads(self):
self.raw_report_processor = ReportsManager(REPORTS_QUEUE)
self.raw_report_processor.start()
self.ping_home_thread = PingHomeThread()
self.ping_home_thread.start()
def run(self):
def signal_handler(*args):
logger.info('Received SIGTERM, shutting down.')
logger.info("Stopping threads, please wait...")
self.stop_threads()
log_path = CONST_FARADAY_HOME_PATH / 'logs' / 'access-logging.log'
site = FaradaySite(self.root_resource,
logPath=log_path,
logFormatter=proxiedLogFormatter)
site.displayTracebacks = False
try:
self.install_signal()
# start threads and processes
self.start_threads()
# web and static content
reactor.listenTCP(server_config.port,
site,
interface=server_config.bind_address)
num_threads = multiprocessing.cpu_count() * 2
logger.info(f'Starting webserver with {num_threads} threads.')
reactor.suggestThreadPoolSize(num_threads)
# websockets
try:
listenWS(self.__build_websockets_resource(),
interface=server_config.bind_address)
except error.CannotListenError:
logger.warn(
'Could not start websockets, address already open. This is ok is you wan to run multiple instances.'
)
except Exception as ex:
logger.warn(f'Could not start websocket, error: {ex}')
logger.info('Faraday Server is ready')
reactor.addSystemEventTrigger('before', 'shutdown', signal_handler)
signal(SIGUSR1, self.restart_threads)
reactor.run()
except error.CannotListenError as e:
logger.error(e)
self.stop_threads()
sys.exit(1)
except Exception as e:
logger.exception(
'Something went wrong when trying to setup the Web UI')
logger.exception(e)
self.stop_threads()
sys.exit(1)
def start_threads(self):
self.raw_report_processor = ReportsManager(REPORTS_QUEUE)
self.raw_report_processor.start()
self.ping_home_thread = PingHomeThread()
self.ping_home_thread.start()