def webserver_serve(app, listener_socket, logfile):
"""Serve app on listener_socket provided.
This sets up process context, including signal handlers, etc,
and runs until signalled.
"""
# create a WSGIServer instance using the listener socket created
server = WSGIServer(listener_socket, app, log=logfile)
server.set_spawn(Pool()) # Make WSGIServer manage worker greenlets
def signal_handler():
# logger.debug('webserver_serve.signal_handler: Worker caught signal. Shutting down.')
app.stop_serving(timeout=HALT_TIMEOUT)
server.stop()
gevent.signal(signal.SIGUSR1, signal_handler)
gevent.signal(signal.SIGINT, signal_handler)
gevent.signal(signal.SIGHUP, signal_handler)
gevent.signal(signal.SIGTERM, signal_handler)
on_initialise.fire()
gevent.spawn_later(1, on_start.fire)
# serve requests
try:
server.serve_forever(stop_timeout=5)
finally:
server.close()
logger.debug('webserver_serve: finished serving')
def run(self):
httpsrv = WSGIServer((self.__listen_ip, int(
self.__listen_port)), self.app.wsgi_app, log=LogLevelChanger)
try:
httpsrv.serve_forever()
except KeyboardInterrupt:
httpsrv.close()
logger.info("Received STOP signal in MITMReceiver")
def wsgi_server():
port = 8886
app = create_app()
http = WSGIServer(('', port), app.wsgi_app)
log.info('Serving on port %s', str(port))
try:
http.serve_forever()
except KeyboardInterrupt:
print('Keyboard interrupt received: EXITING')
finally:
http.close()
log.info('Server terminated!')
def start_server(port):
assert type(port) == int
try:
pool = Pool(256)
server = WSGIServer(('', port), app, spawn=pool)
server.serve_forever()
except Exception as e:
print(e)
server.close()
del pool
del server
finally:
server.close()
del pool
del server
print("Closed Server")
class ServerThread(Thread):
def __init__(self, *args, port, app, host, **kwargs):
super().__init__(*args, **kwargs)
self.port = port
self.app = app
self.host = host
self.server: WSGIServer = None
def run(self):
self.server = WSGIServer((self.host, self.port), self.app)
self.server.serve_forever()
def close(self):
self.server.close()
def url(self):
return 'http://' + self.host + ':' + str(self.port)
class ServerThread(threading.Thread):
def __init__(self, app, port=5005):
threading.Thread.__init__(self)
self.port = port
self.srv = WSGIServer(("0.0.0.0", self.port), app)
self.ctx = app.app_context()
self.ctx.push()
def run(self):
logging.info('Starting threaded server.')
self.srv.serve_forever()
def shutdown(self):
logging.info('Stopping threaded server.')
self.srv.stop()
self.srv.close()
del self
class HTTPServer:
def __init__(self):
self.Handler = HTTPHandler
self.listener = None
def start(self, host, port):
conn = (host, port)
self.listener = WSGIServer(conn, self.Handler)
try:
logger.info('Starting HTTPServer on {}'.format(conn))
self.listener.serve_forever()
except:
logger.debug('Exception Occurred!')
finally:
self.stop()
def stop(self):
self.listener.close()
class EmbargoServer(threading.Thread):
def __init__(self, host: str = 'localhost', port: int = 7777):
super().__init__(daemon=True)
self._host = host
self._port = port
self._server = None
def run(self) -> None:
self._server = WSGIServer((self._host, self._port), server_app)
self._server.serve_forever()
def stop(self) -> None:
if self._server is None:
return
self._server.stop()
self._server.close()
class WebServer(threading.Thread):
def __init__(self):
super().__init__()
def run(self):
self.server = WSGIServer(('0.0.0.0', 80), saveInfo_app)
self.gevent_signal = gevent.hub.signal(signal.SIGTERM, self.shutdown)
self.server.serve_forever()
# ======================== for development only =====================
# def run(self):
# saveInfo_app.run(host='0.0.0.0', port=7497, debug=False) # run collecting app
# ===================================================================
def shutdown(
self
): # SIGINT or SIGTERM doesn't really matter since what shutdown server stays here
print(f'Shutting down server...\n')
self.server.stop()
self.server.close()
self.gevent_signal.cancel()
def run_receiver(self, listen_ip, listen_port, mitm_mapper, args_passed,
auths_passed):
global application_args, auths
application_args = args_passed
auths = auths_passed
self.__listen_ip = listen_ip
self.__listen_port = listen_port
self.__mitm_mapper: MitmMapper = mitm_mapper
self.app = Flask("MITMReceiver")
self.add_endpoint(endpoint='/',
endpoint_name='receive_protos',
handler=self.proto_endpoint,
methods_passed=['POST'])
self.add_endpoint(endpoint='/get_latest_mitm/',
endpoint_name='get_latest_mitm/',
handler=self.get_latest,
methods_passed=['GET'])
self.add_endpoint(endpoint='/get_addresses/',
endpoint_name='get_addresses/',
handler=self.get_addresses,
methods_passed=['GET'])
self._data_queue: JoinableQueue = JoinableQueue()
self._db_wrapper = DbFactory.get_wrapper(args_passed)
self.worker_threads = []
for i in range(application_args.mitmreceiver_data_workers):
data_processor: MitmDataProcessor = MitmDataProcessor()
t = Process(name='MITMReceiver-%s' % str(i),
target=data_processor.received_data_worker,
args=(data_processor, self._data_queue,
application_args, self.__mitm_mapper))
t.start()
self.worker_threads.append(t)
httpsrv = WSGIServer((self.__listen_ip, int(self.__listen_port)),
self.app.wsgi_app,
log=LogLevelChanger)
try:
httpsrv.serve_forever()
except KeyboardInterrupt as e:
logger.info("Stopping MITMReceiver")
httpsrv.close()
class BackendServer:
def __init__(self, host: str, port: int):
self.host = host
self.port = port
# create the flask app and the wsgi server
self._app = self._create_flask_app()
self._wsgi_server = None # has to be set inside the _run_server function
self._thread = Thread(target=self._run_server)
@staticmethod
def _create_flask_app() -> Flask:
"""
Helper function to create the flask app with the appropriate routes.
:return: Instance of the Flask route handling object.
"""
app = Flask(__name__)
CORS(app)
@app.route("/")
def index():
return "WebApp Index"
@app.route("/api/add_user", methods=["POST"])
def add_user():
store = InMemoryDataStore()
user_json = request.get_json(force=True)
store.add_user(firstname=user_json["firstname"],
lastname=user_json["lastname"],
age=user_json["age"],
email=user_json["email"])
return jsonify("Success")
@app.route("/api/update_user_portfolio", methods=["POST"])
def update_user_portfolio():
store = InMemoryDataStore()
json_data = request.get_json(force=True)
cash_json = json_data["cashAssets"]
cash = CashAssets(
cash_json["checkingAcs"], cash_json["savingsAcs"],
cash_json["moneyMarketAccounts"], cash_json["savingsBonds"],
cash_json["cds"], cash_json["lifeInsurance"],
Other(cash_json["other"]["title"],
cash_json["other"]["amount"]))
invested_json = json_data["investedAssets"]
invested = InvestedAssets(
invested_json["brokerage"],
Other(invested_json["otherTax"]["title"],
invested_json["otherTax"]["amount"]),
invested_json["ira"], invested_json["rothIra"],
invested_json["k401"], invested_json["sepIra"],
invested_json["keogh"], invested_json["pension"],
invested_json["annuity"], invested_json["realEstate"],
invested_json["soleProp"], invested_json["partnership"],
invested_json["cCorp"], invested_json["sCorp"],
invested_json["limitedLiabilityCompany"],
Other(invested_json["otherBusiness"]["title"],
invested_json["otherBusiness"]["amount"]))
use_json = json_data["useAssets"]
use = UseAssets(
use_json["principalHome"], use_json["vacationHome"],
use_json["vehicles"], use_json["homeFurnishings"],
use_json["artsAndAntiques"], use_json["jewelryAndFurs"],
Other(use_json["other"]["title"], use_json["other"]["amount"]))
current_json = json_data["currentLiabilities"]
current = CurrentLiabilities(
current_json["creditCardBalance"],
current_json["incomeTaxOwed"],
Other(current_json["other"]["title"],
current_json["other"]["amount"]))
long_json = json_data["longTermLiabilities"]
long = LongTermLiabilities(
long_json["homeMortgage"], long_json["homeEquityLoan"],
long_json["rentPropertiesMortgage"], long_json["carLoans"],
long_json["studentLoans"],
long_json["lifeInsurancePolicyLoans"],
Other(long_json["other"]["title"],
long_json["other"]["amount"]))
currency = Currencies[json_data["currency"]]
email = json_data["email"]
store.update_user_portfolio(email=email,
portfolio=Portfolio(
currency, cash, invested, use,
current, long))
return jsonify("Success")
@app.route("/api/get_user")
def get_user():
# expecting query arguments "/api/[email protected]"
email = request.args.get("email")
store = InMemoryDataStore()
u = store.get_user(
email) # will raise an error if not found resulting in a 500
return jsonify(u.to_json())
@app.route("/api/get_net_worth")
def get_net_worth():
# expecting query arguments "/api/[email protected]"
email = request.args.get("email")
store = InMemoryDataStore()
u = store.get_user(
email) # will raise an error if not found resulting in a 500
return jsonify(u.net_worth)
@app.route("/api/get_assets")
def get_assets():
# expecting query arguments "/api/[email protected]"
email = request.args.get("email")
store = InMemoryDataStore()
assets = store.get_user_assets(
email) # will raise an error if not found resulting in a 500
return jsonify(assets)
@app.route("/api/get_liabilities")
def get_liabilities():
# expecting query arguments "/api/[email protected]"
email = request.args.get("email")
store = InMemoryDataStore()
liabilities = store.get_user_liabilities(
email) # will raise an error if not found resulting in a 500
return jsonify(liabilities)
return app
def _run_server(self):
"""
Helper function to start the wsgi server in a separate thread, using the
pointers to the instantiated WSGI server from the constructor.
"""
self._wsgi_server = WSGIServer((self.host, self.port), self._app)
self._wsgi_server.serve_forever()
def start(self):
if not self._thread.is_alive():
self._thread.start()
print(f"Started BackendServer on {self.host}:{self.port}")
else:
print("The BackendServer is already running")
def stop(self, timeout=1):
if self._thread.is_alive() and self._wsgi_server is not None:
# kill the WSGI server and then kill the thread
self._wsgi_server.stop(timeout=timeout)
self._wsgi_server.close()
self._thread.join(timeout=timeout + 1)
# sleep to give the server time to kill requests
time.sleep(1)
if self._thread.is_alive():
raise OSError("Server Thread did not shutdown cleanly")
else:
print("BackendServer successfully shutdown")
else:
print("The BackendServer is already shutdown")
class server:
wsgiserver = None
restart= False
def __init__(self):
signal.signal(signal.SIGINT, self.killServer)
signal.signal(signal.SIGTERM, self.killServer)
def start_gevent(self):
try:
ssl_args = dict()
certfile_path = web.ub.config.get_config_certfile()
keyfile_path = web.ub.config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path):
ssl_args = {"certfile": certfile_path,
"keyfile": keyfile_path}
else:
web.app.logger.info('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s' % (certfile_path, keyfile_path))
if os.name == 'nt':
self.wsgiserver= WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
else:
self.wsgiserver = WSGIServer(('', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
web.py3_gevent_link = self.wsgiserver
self.wsgiserver.serve_forever()
except SocketError:
try:
web.app.logger.info('Unable to listen on \'\', trying on IPv4 only...')
self.wsgiserver = WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
web.py3_gevent_link = self.wsgiserver
self.wsgiserver.serve_forever()
except (OSError, SocketError) as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
except Exception:
web.app.logger.info("Unknown error while starting gevent")
def startServer(self):
if gevent_present:
web.app.logger.info('Starting Gevent server')
# leave subprocess out to allow forking for fetchers and processors
self.start_gevent()
else:
try:
ssl = None
web.app.logger.info('Starting Tornado server')
certfile_path = web.ub.config.get_config_certfile()
keyfile_path = web.ub.config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path):
ssl = {"certfile": certfile_path,
"keyfile": keyfile_path}
else:
web.app.logger.info('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s' % (certfile_path, keyfile_path))
# Max Buffersize set to 200MB
http_server = HTTPServer(WSGIContainer(web.app),
max_buffer_size = 209700000,
ssl_options=ssl)
http_server.listen(web.ub.config.config_port)
self.wsgiserver=IOLoop.instance()
self.wsgiserver.start()
# wait for stop signal
self.wsgiserver.close(True)
except SocketError as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
# ToDo: Somehow caused by circular import under python3 refactor
if sys.version_info > (3, 0):
self.restart = web.py3_restart_Typ
if self.restart == True:
web.app.logger.info("Performing restart of Calibre-Web")
web.helper.global_WorkerThread.stop()
if os.name == 'nt':
arguments = ["\"" + sys.executable + "\""]
for e in sys.argv:
arguments.append("\"" + e + "\"")
os.execv(sys.executable, arguments)
else:
os.execl(sys.executable, sys.executable, *sys.argv)
else:
web.app.logger.info("Performing shutdown of Calibre-Web")
web.helper.global_WorkerThread.stop()
sys.exit(0)
def setRestartTyp(self,starttyp):
self.restart = starttyp
# ToDo: Somehow caused by circular import under python3 refactor
web.py3_restart_Typ = starttyp
def killServer(self, signum, frame):
self.stopServer()
def stopServer(self):
# ToDo: Somehow caused by circular import under python3 refactor
if sys.version_info > (3, 0):
if not self.wsgiserver:
if gevent_present:
self.wsgiserver = web.py3_gevent_link
else:
self.wsgiserver = IOLoop.instance()
if self.wsgiserver:
if gevent_present:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback(self.wsgiserver.stop)
@staticmethod
def getNameVersion():
if gevent_present:
return {'Gevent':'v'+geventVersion}
else:
return {'Tornado':'v'+tornadoVersion}
def test_low_level_echo(self):
data = []
data.append("")
data.append("0123456789")
data.append("0123456789" * 20)
data.append("0123456789" * 2000)
data.append("0123456789" * 20000)
data.append("0123456789" * 200000)
data.append(b"")
data.append(b"0123456789")
data.append(b"0123456789" * 20)
data.append(b"0123456789" * 2000)
data.append(b"0123456789" * 20000)
data.append(b"0123456789" * 200000)
def test_echo_actual(environ, start_response):
try:
websocket = environ.get("wsgi.websocket")
if websocket is None:
return http_handler(environ, start_response)
while True:
message = websocket.receive()
if message is None:
break
websocket.send(message)
websocket.close()
except Exception as e:
self.err.append(e)
server = WSGIServer(self.s_socket, application=test_echo_actual, handler_class=TestWebSocketHandler)
server.logger = getLogger(__name__)
server.start()
self.server = server
try:
self.server_thread = Thread(target=self.start_server)
self.server_thread.start()
ws = create_connection("ws://%s:%s/" % (self.server.environ["SERVER_NAME"],
self.server.environ["SERVER_PORT"]))
try:
for i in range(2):
if i == 1:
ws.set_mask_key(lambda _: b'1234')
for d in data:
if isinstance(d, bytes):
ws.send_binary(d)
else:
ws.send(d)
self.assertEqual(ws.recv(), d)
ws.ping("ping")
pong = ws.recv_data_frame(10)
self.assertEqual(pong[0], pong[1].OPCODE_PONG)
self.assertEqual(pong[1].data, b'ping')
ping = ws.recv_data_frame(9)
self.assertEqual(ping[0], ping[1].OPCODE_PING)
self.assertEqual(ping[1].data, b'')
ws.pong(ping[1].data)
ping = ws.recv_data_frame(9)
self.assertEqual(ping[0], ping[1].OPCODE_PING)
self.assertEqual(ping[1].data, b'')
ws.pong(ping[1].data)
ping = ws.recv_data_frame(9)
self.assertEqual(ping[0], ping[1].OPCODE_PING)
self.assertEqual(ping[1].data, b'ping')
ws.pong(ping[1].data)
ping = ws.recv_data_frame(9)
self.assertEqual(ping[0], ping[1].OPCODE_PING)
self.assertEqual(ping[1].data, b'ping')
ws.pong(ping[1].data)
self.assertFalse(self.err)
finally:
ws.close(timeout=10)
finally:
server.close()
class server:
wsgiserver = None
restart = False
def __init__(self):
pass
def start_gevent(self):
try:
ssl_args = dict()
if web.ub.config.get_config_certfile(
) and web.ub.config.get_config_keyfile():
ssl_args = {
"certfile": web.ub.config.get_config_certfile(),
"keyfile": web.ub.config.get_config_keyfile()
}
if os.name == 'nt':
self.wsgiserver = WSGIServer(
('0.0.0.0', web.ub.config.config_port),
web.app,
spawn=Pool(),
**ssl_args)
else:
self.wsgiserver = WSGIServer(('', web.ub.config.config_port),
web.app,
spawn=Pool(),
**ssl_args)
self.wsgiserver.serve_forever()
except SocketError:
web.app.logger.info(
'Unable to listen on \'\', trying on IPv4 only...')
self.wsgiserver = WSGIServer(
('0.0.0.0', web.ub.config.config_port),
web.app,
spawn=Pool(),
**ssl_args)
self.wsgiserver.serve_forever()
except Exception:
web.app.logger.info("Unknown error while starting gevent")
def startServer(self):
if gevent_present:
web.app.logger.info('Starting Gevent server')
# leave subprocess out to allow forking for fetchers and processors
self.start_gevent()
else:
web.app.logger.info('Starting Tornado server')
if web.ub.config.get_config_certfile(
) and web.ub.config.get_config_keyfile():
ssl = {
"certfile": web.ub.config.get_config_certfile(),
"keyfile": web.ub.config.get_config_keyfile()
}
else:
ssl = None
# Max Buffersize set to 200MB
http_server = HTTPServer(WSGIContainer(web.app),
max_buffer_size=209700000,
ssl_options=ssl)
http_server.listen(web.ub.config.config_port)
self.wsgiserver = IOLoop.instance()
self.wsgiserver.start() # wait for stop signal
self.wsgiserver.close(True)
if self.restart == True:
web.app.logger.info("Performing restart of Calibre-web")
web.helper.global_WorkerThread.stop()
if os.name == 'nt':
arguments = ["\"" + sys.executable + "\""]
for e in sys.argv:
arguments.append("\"" + e + "\"")
os.execv(sys.executable, arguments)
else:
os.execl(sys.executable, sys.executable, *sys.argv)
else:
web.app.logger.info("Performing shutdown of Calibre-web")
web.helper.global_WorkerThread.stop()
sys.exit(0)
def setRestartTyp(self, starttyp):
self.restart = starttyp
def stopServer(self):
if gevent_present:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback(self.wsgiserver.stop)
@staticmethod
def getNameVersion():
if gevent_present:
return {'Gevent': 'v' + geventVersion}
else:
return {'Tornado': 'v' + tornadoVersion}
class WebServer(object):
def __init__(self):
signal.signal(signal.SIGINT, self._killServer)
signal.signal(signal.SIGTERM, self._killServer)
self.wsgiserver = None
self.access_logger = None
self.restart = False
self.app = None
self.listen_address = None
self.listen_port = None
self.unix_socket_file = None
self.ssl_args = None
def init_app(self, application, config):
self.app = application
self.listen_address = config.get_config_ipaddress()
self.listen_port = config.config_port
if config.config_access_log:
log_name = "gevent.access" if _GEVENT else "tornado.access"
formatter = logger.ACCESS_FORMATTER_GEVENT if _GEVENT else logger.ACCESS_FORMATTER_TORNADO
self.access_logger, logfile = logger.create_access_log(config.config_access_logfile, log_name, formatter)
if logfile != config.config_access_logfile:
log.warning("Accesslog path %s not valid, falling back to default", config.config_access_logfile)
config.config_access_logfile = logfile
config.save()
else:
if not _GEVENT:
logger.get('tornado.access').disabled = True
certfile_path = config.get_config_certfile()
keyfile_path = config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path):
self.ssl_args = dict(certfile=certfile_path, keyfile=keyfile_path)
else:
log.warning('The specified paths for the ssl certificate file and/or key file seem to be broken. '
'Ignoring ssl.')
log.warning('Cert path: %s', certfile_path)
log.warning('Key path: %s', keyfile_path)
def _make_gevent_unix_socket(self, socket_file):
# the socket file must not exist prior to bind()
if os.path.exists(socket_file):
# avoid nuking regular files and symbolic links (could be a mistype or security issue)
if os.path.isfile(socket_file) or os.path.islink(socket_file):
raise OSError(errno.EEXIST, os.strerror(errno.EEXIST), socket_file)
os.remove(socket_file)
unix_sock = WSGIServer.get_listener(socket_file, family=socket.AF_UNIX)
self.unix_socket_file = socket_file
# ensure current user and group have r/w permissions, no permissions for other users
# this way the socket can be shared in a semi-secure manner
# between the user running calibre-web and the user running the fronting webserver
os.chmod(socket_file, 0o660)
return unix_sock
def _make_gevent_socket(self):
if os.name != 'nt':
unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET")
if unix_socket_file:
return self._make_gevent_unix_socket(unix_socket_file), "unix:" + unix_socket_file
if self.listen_address:
return (self.listen_address, self.listen_port), None
if os.name == 'nt':
self.listen_address = '0.0.0.0'
return (self.listen_address, self.listen_port), None
try:
address = ('::', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET6)
except socket.error as ex:
log.error('%s', ex)
log.warning('Unable to listen on "", trying on IPv4 only...')
address = ('', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET)
return sock, _readable_listen_address(*address)
@staticmethod
def _get_args_for_reloading():
"""Determine how the script was executed, and return the args needed
to execute it again in a new process.
Code from https://github.com/pyload/pyload. Author GammaC0de, voulter
"""
rv = [sys.executable]
py_script = sys.argv[0]
args = sys.argv[1:]
# Need to look at main module to determine how it was executed.
__main__ = sys.modules["__main__"]
# The value of __package__ indicates how Python was called. It may
# not exist if a setuptools script is installed as an egg. It may be
# set incorrectly for entry points created with pip on Windows.
if getattr(__main__, "__package__", None) is None or (
os.name == "nt"
and __main__.__package__ == ""
and not os.path.exists(py_script)
and os.path.exists("{}.exe".format(py_script))
):
# Executed a file, like "python app.py".
py_script = os.path.abspath(py_script)
if os.name == "nt":
# Windows entry points have ".exe" extension and should be
# called directly.
if not os.path.exists(py_script) and os.path.exists("{}.exe".format(py_script)):
py_script += ".exe"
if (
os.path.splitext(sys.executable)[1] == ".exe"
and os.path.splitext(py_script)[1] == ".exe"
):
rv.pop(0)
rv.append(py_script)
else:
# Executed a module, like "python -m module".
if sys.argv[0] == "-m":
args = sys.argv
else:
if os.path.isfile(py_script):
# Rewritten by Python from "-m script" to "/path/to/script.py".
py_module = __main__.__package__
name = os.path.splitext(os.path.basename(py_script))[0]
if name != "__main__":
py_module += ".{}".format(name)
else:
# Incorrectly rewritten by pydevd debugger from "-m script" to "script".
py_module = py_script
rv.extend(("-m", py_module.lstrip(".")))
rv.extend(args)
return rv
def _start_gevent(self):
ssl_args = self.ssl_args or {}
try:
sock, output = self._make_gevent_socket()
if output is None:
output = _readable_listen_address(self.listen_address, self.listen_port)
log.info('Starting Gevent server on %s', output)
self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, spawn=Pool(), **ssl_args)
if ssl_args:
wrap_socket = self.wsgiserver.wrap_socket
def my_wrap_socket(*args, **kwargs):
try:
return wrap_socket(*args, **kwargs)
except (ssl.SSLError, OSError) as ex:
log.warning('Gevent SSL Error: %s', ex)
raise GreenletExit
self.wsgiserver.wrap_socket = my_wrap_socket
self.wsgiserver.serve_forever()
finally:
if self.unix_socket_file:
os.remove(self.unix_socket_file)
self.unix_socket_file = None
def _start_tornado(self):
if os.name == 'nt' and sys.version_info > (3, 7):
import asyncio
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
log.info('Starting Tornado server on %s', _readable_listen_address(self.listen_address, self.listen_port))
# Max Buffersize set to 200MB )
http_server = HTTPServer(WSGIContainer(self.app),
max_buffer_size=209700000,
ssl_options=self.ssl_args)
http_server.listen(self.listen_port, self.listen_address)
self.wsgiserver = IOLoop.current()
self.wsgiserver.start()
# wait for stop signal
self.wsgiserver.close(True)
def start(self):
try:
if _GEVENT:
# leave subprocess out to allow forking for fetchers and processors
self._start_gevent()
else:
self._start_tornado()
except Exception as ex:
log.error("Error starting server: %s", ex)
print("Error starting server: %s" % ex)
self.stop()
return False
finally:
self.wsgiserver = None
if not self.restart:
log.info("Performing shutdown of Calibre-Web")
# prevent irritiating log of pending tasks message from asyncio
logger.get('asyncio').setLevel(logger.logging.CRITICAL)
return True
log.info("Performing restart of Calibre-Web")
args = self._get_args_for_reloading()
subprocess.call(args, close_fds=True)
return True
def _killServer(self, __, ___):
self.stop()
def stop(self, restart=False):
from . import updater_thread
updater_thread.stop()
log.info("webserver stop (restart=%s)", restart)
self.restart = restart
if self.wsgiserver:
if _GEVENT:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback_from_signal(self.wsgiserver.stop)
class WebService(Bottle):
def __init__(self, config, car):
super(WebService, self).__init__()
self.log = logging.getLogger("EVNotiPi/WebService")
self.log.info("Initializing WebService")
self.car = car
self.data = {}
self.data_lock = Condition()
self.running = False
self.server = None
self.thread = None
self.route('/data/live/ws', callback=self.handle_websocket)
self.route('/data', callback=self.handle_data)
self.route('/', callback=self.handle_index)
self.route('/static/<filename>', callback=self.handle_static)
def handle_websocket(self):
wsock = request.environ.get('wsgi.websocket')
if not wsock:
abort(400, 'Expected WebSocket request.')
while True:
try:
with self.data_lock:
self.data_lock.wait()
data = self.data
data = json.dumps(data)
wsock.send(data)
except WebSocketError:
break
def handle_index(self):
return static_file('index.html', root="./web")
def handle_static(self, filename):
return static_file(filename, root="./web")
def handle_data(self):
return json.dumps(self.data)
def start(self):
self.running = True
self.server = WSGIServer(('::', 8080),
self,
handler_class=WebSocketHandler)
self.thread = Thread(target=self.server.serve_forever)
self.thread.start()
self.car.register_data(self.data_callback)
def stop(self):
self.car.unregister_data(self.data_callback)
self.running = False
self.server.stop()
self.server.close()
self.thread.join()
def data_callback(self, data):
with self.data_lock:
self.data = data
self.data_lock.notify_all()
def check_thread(self):
return self.thread.is_alive()
class server:
wsgiserver = None
restart = False
def __init__(self):
signal.signal(signal.SIGINT, self.killServer)
signal.signal(signal.SIGTERM, self.killServer)
def start_gevent(self):
try:
ssl_args = dict()
certfile_path = web.ub.config.get_config_certfile()
keyfile_path = web.ub.config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(
keyfile_path):
ssl_args = {
"certfile": certfile_path,
"keyfile": keyfile_path
}
else:
web.app.logger.info(
'The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s'
% (certfile_path, keyfile_path))
if os.name == 'nt':
self.wsgiserver = WSGIServer(
('0.0.0.0', web.ub.config.config_port),
web.app,
spawn=Pool(),
**ssl_args)
else:
self.wsgiserver = WSGIServer(('', web.ub.config.config_port),
web.app,
spawn=Pool(),
**ssl_args)
web.py3_gevent_link = self.wsgiserver
self.wsgiserver.serve_forever()
except SocketError:
try:
web.app.logger.info(
'Unable to listen on \'\', trying on IPv4 only...')
self.wsgiserver = WSGIServer(
('0.0.0.0', web.ub.config.config_port),
web.app,
spawn=Pool(),
**ssl_args)
web.py3_gevent_link = self.wsgiserver
self.wsgiserver.serve_forever()
except (OSError, SocketError) as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
except Exception:
web.app.logger.info("Unknown error while starting gevent")
def startServer(self):
if gevent_present:
web.app.logger.info('Starting Gevent server')
# leave subprocess out to allow forking for fetchers and processors
self.start_gevent()
else:
try:
ssl = None
web.app.logger.info('Starting Tornado server')
certfile_path = web.ub.config.get_config_certfile()
keyfile_path = web.ub.config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(
keyfile_path):
ssl = {
"certfile": certfile_path,
"keyfile": keyfile_path
}
else:
web.app.logger.info(
'The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s'
% (certfile_path, keyfile_path))
# Max Buffersize set to 200MB
http_server = HTTPServer(WSGIContainer(web.app),
max_buffer_size=209700000,
ssl_options=ssl)
http_server.listen(web.ub.config.config_port)
self.wsgiserver = IOLoop.instance()
web.py3_gevent_link = self.wsgiserver
self.wsgiserver.start()
# wait for stop signal
self.wsgiserver.close(True)
except SocketError as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
# ToDo: Somehow caused by circular import under python3 refactor
if sys.version_info > (3, 0):
self.restart = web.py3_restart_Typ
if self.restart == True:
web.app.logger.info("Performing restart of Calibre-Web")
web.helper.global_WorkerThread.stop()
if os.name == 'nt':
arguments = ["\"" + sys.executable + "\""]
for e in sys.argv:
arguments.append("\"" + e + "\"")
os.execv(sys.executable, arguments)
else:
os.execl(sys.executable, sys.executable, *sys.argv)
else:
web.app.logger.info("Performing shutdown of Calibre-Web")
web.helper.global_WorkerThread.stop()
sys.exit(0)
def setRestartTyp(self, starttyp):
self.restart = starttyp
# ToDo: Somehow caused by circular import under python3 refactor
web.py3_restart_Typ = starttyp
def killServer(self, signum, frame):
self.stopServer()
def stopServer(self):
# ToDo: Somehow caused by circular import under python3 refactor
if sys.version_info > (3, 0):
if not self.wsgiserver:
# if gevent_present:
self.wsgiserver = web.py3_gevent_link
#else:
# self.wsgiserver = IOLoop.instance()
if self.wsgiserver:
if gevent_present:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback(self.wsgiserver.stop)
@staticmethod
def getNameVersion():
if gevent_present:
return {'Gevent': 'v' + geventVersion}
else:
return {'Tornado': 'v' + tornadoVersion}
from gevent.pywsgi import WSGIServer
from geventwebsocket.handler import WebSocketHandler
from Twidder import app
print(">>Starting server<<")
app.debug = True
http_server = WSGIServer(('', 5000), app, handler_class=WebSocketHandler)
try:
http_server.serve_forever()
except KeyboardInterrupt:
print ">>Shutting down server<<"
http_server.close()
def close(self):
if self.closed:
sys.exit('Multiple exit signals received - aborting.')
else:
log('Closing listener socket')
WSGIServer.close(self)
class Server:
"""
REST API server
"""
def __init__(self):
self.process = None
self.app = Flask(__name__)
self.app.config['MAX_CONTENT_LENGTH'] = 2 * 1024
self.app.url_map.strict_slashes = False
self.api = Api(self.app)
# initialize SSL context
self.context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self.context.verify_mode = ssl.CERT_REQUIRED
self.context.set_ciphers(':'.join(TLS_CIPHERS))
# allow TLS 1.2 and later
self.context.options |= ssl.OP_NO_SSLv2
self.context.options |= ssl.OP_NO_SSLv3
self.context.options |= ssl.OP_NO_TLSv1
self.context.options |= ssl.OP_NO_TLSv1_1
# Apps and Pools API
self.api.add_resource(Apps, '/apps')
self.api.add_resource(App, '/apps/<int:app_id>')
self.api.add_resource(Pools, '/pools')
self.api.add_resource(Pool, '/pools/<int:pool_id>')
# SST-CP API
if caps.sstcp_enabled():
self.api.add_resource(Powers, '/power_profiles')
self.api.add_resource(Power, '/power_profiles/<int:profile_id>')
# Stats and Capabilities API
self.api.add_resource(Stats, '/stats')
self.api.add_resource(Caps, '/caps')
# SST-BF API
if caps.sstbf_enabled():
self.api.add_resource(Sstbf, '/caps/sstbf')
# RDT interface API
self.api.add_resource(CapsRdtIface, '/caps/rdt_iface')
# MBA API
if caps.mba_supported():
self.api.add_resource(CapsMba, '/caps/mba')
self.api.add_resource(CapsMbaCtrl, '/caps/mba_ctrl')
# L3 CAT API
if caps.cat_l3_supported():
self.api.add_resource(CapsL3ca, '/caps/' + common.CAT_L3_CAP)
# L2 CAT API
if caps.cat_l2_supported():
self.api.add_resource(CapsL2ca, '/caps/' + common.CAT_L2_CAP)
# Reset API
self.api.add_resource(Reset, '/reset')
self.app.register_error_handler(HTTPException, Server.error_handler)
def start(self, host, port, debug=False):
"""
Start REST server
Parameters:
host: address to bind to
port: port to bind to
debug(bool): Debug flag
Returns:
0 on success
"""
try:
# check for file existence and type
with open(TLS_CERT_FILE, opener=common.check_link):
pass
with open(TLS_KEY_FILE, opener=common.check_link):
pass
self.context.load_cert_chain(TLS_CERT_FILE, TLS_KEY_FILE)
except (FileNotFoundError, PermissionError) as ex:
log.error("SSL cert or key file, {}".format(str(ex)))
return -1
# loading CA crt file - it is needed for mTLS verification of client certificates
try:
with open(TLS_CA_CERT_FILE, opener=common.check_link):
pass
self.context.load_verify_locations(cafile=TLS_CA_CERT_FILE)
except (FileNotFoundError, PermissionError) as ex:
log.error("CA certificate file, {}".format(str(ex)))
return -1
self.http_server = WSGIServer((host, port),
self.app,
ssl_context=self.context,
spawn=1)
def handle_gevent_stop(signum, frame):
log.info("Stopping gevent server loop")
self.http_server.stop()
self.http_server.close()
# dedicated handler for gevent server is needed to stop it gracefully
# before process will be terminated
signal.signal(signal.SIGINT, handle_gevent_stop)
self.process = multiprocessing.Process(
target=self.http_server.serve_forever)
self.process.start()
return 0
def terminate(self):
"""
Terminates server
"""
os.kill(self.process.pid, signal.SIGINT)
sleep(1)
if self.process.is_alive():
self.process.terminate()
self.process.join()
@staticmethod
def error_handler(error):
"""
Error handler
Parameters:
error: error
"""
common.STATS_STORE.general_stats_inc_num_err()
response = {'message': error.description}
return json.dumps(response), error.code
except:
return jsonify({"message": "server error"}), 500
@app.route('/login', methods=['POST'])
def login():
try:
auth = request.authorization
if not auth or not auth.username or not auth.password:
return jsonify({'message': 'invalid token'}), 401
token = authHandler.login(auth)
return jsonify({'token': token.decode('UTF-8')})
except TokenVerificationException:
return jsonify({'message': 'token verification failed'}), 401
except:
return jsonify({"message": "login failed"}), 401
################# Init Server #################
if __name__ == '__main__':
# set up and run server
# handleArgumentInputs(sys.argv[1:])
server = WSGIServer(('0.0.0.0', int(8080)), app)
try:
print("Server is ready...")
server.serve_forever()
except KeyboardInterrupt:
print("Server Stopped")
server.close()
class WebServer(object):
def __init__(self):
signal.signal(signal.SIGINT, self._killServer)
signal.signal(signal.SIGTERM, self._killServer)
self.wsgiserver = None
self.access_logger = None
self.restart = False
self.app = None
self.listen_address = None
self.listen_port = None
self.unix_socket_file = None
self.ssl_args = None
def init_app(self, application, config):
self.app = application
self.listen_address = config.get_config_ipaddress()
self.listen_port = config.config_port
if config.config_access_log:
log_name = "gevent.access" if _GEVENT else "tornado.access"
formatter = logger.ACCESS_FORMATTER_GEVENT if _GEVENT else logger.ACCESS_FORMATTER_TORNADO
self.access_logger, logfile = logger.create_access_log(
config.config_access_logfile, log_name, formatter)
if logfile != config.config_access_logfile:
log.warning(
"Accesslog path %s not valid, falling back to default",
config.config_access_logfile)
config.config_access_logfile = logfile
config.save()
else:
if not _GEVENT:
logger.get('tornado.access').disabled = True
certfile_path = config.get_config_certfile()
keyfile_path = config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path):
self.ssl_args = dict(certfile=certfile_path,
keyfile=keyfile_path)
else:
log.warning(
'The specified paths for the ssl certificate file and/or key file seem to be broken. '
'Ignoring ssl.')
log.warning('Cert path: %s', certfile_path)
log.warning('Key path: %s', keyfile_path)
def _make_gevent_unix_socket(self, socket_file):
# the socket file must not exist prior to bind()
if os.path.exists(socket_file):
# avoid nuking regular files and symbolic links (could be a mistype or security issue)
if os.path.isfile(socket_file) or os.path.islink(socket_file):
raise OSError(errno.EEXIST, os.strerror(errno.EEXIST),
socket_file)
os.remove(socket_file)
unix_sock = WSGIServer.get_listener(socket_file, family=socket.AF_UNIX)
self.unix_socket_file = socket_file
# ensure current user and group have r/w permissions, no permissions for other users
# this way the socket can be shared in a semi-secure manner
# between the user running calibre-web and the user running the fronting webserver
os.chmod(socket_file, 0o660)
return unix_sock
def _make_gevent_socket(self):
if os.name != 'nt':
unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET")
if unix_socket_file:
return self._make_gevent_unix_socket(
unix_socket_file), "unix:" + unix_socket_file
if self.listen_address:
return (self.listen_address, self.listen_port), None
if os.name == 'nt':
self.listen_address = '0.0.0.0'
return (self.listen_address, self.listen_port), None
try:
address = ('::', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET6)
except socket.error as ex:
log.error('%s', ex)
log.warning('Unable to listen on "", trying on IPv4 only...')
address = ('', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET)
return sock, _readable_listen_address(*address)
def _start_gevent(self):
ssl_args = self.ssl_args or {}
try:
sock, output = self._make_gevent_socket()
if output is None:
output = _readable_listen_address(self.listen_address,
self.listen_port)
log.info('Starting Gevent server on %s', output)
self.wsgiserver = WSGIServer(sock,
self.app,
log=self.access_logger,
spawn=Pool(),
**ssl_args)
if ssl_args:
wrap_socket = self.wsgiserver.wrap_socket
def my_wrap_socket(*args, **kwargs):
try:
return wrap_socket(*args, **kwargs)
except (ssl.SSLError) as ex:
log.warning('Gevent SSL Error: %s', ex)
raise GreenletExit
self.wsgiserver.wrap_socket = my_wrap_socket
self.wsgiserver.serve_forever()
finally:
if self.unix_socket_file:
os.remove(self.unix_socket_file)
self.unix_socket_file = None
def _start_tornado(self):
if os.name == 'nt' and sys.version_info > (3, 7):
import asyncio
asyncio.set_event_loop_policy(
asyncio.WindowsSelectorEventLoopPolicy())
log.info(
'Starting Tornado server on %s',
_readable_listen_address(self.listen_address, self.listen_port))
# Max Buffersize set to 200MB )
http_server = HTTPServer(WSGIContainer(self.app),
max_buffer_size=209700000,
ssl_options=self.ssl_args)
http_server.listen(self.listen_port, self.listen_address)
self.wsgiserver = IOLoop.current()
self.wsgiserver.start()
# wait for stop signal
self.wsgiserver.close(True)
def start(self):
try:
if _GEVENT:
# leave subprocess out to allow forking for fetchers and processors
self._start_gevent()
else:
self._start_tornado()
except Exception as ex:
log.error("Error starting server: %s", ex)
print("Error starting server: %s" % ex)
self.stop()
return False
finally:
self.wsgiserver = None
if not self.restart:
log.info("Performing shutdown of Calibre-Web")
# prevent irritiating log of pending tasks message from asyncio
logger.get('asyncio').setLevel(logger.logging.CRITICAL)
return True
log.info("Performing restart of Calibre-Web")
arguments = list(sys.argv)
arguments.insert(0, sys.executable)
if os.name == 'nt':
arguments = ["\"%s\"" % a for a in arguments]
os.execv(sys.executable, arguments)
return True
def _killServer(self, __, ___):
self.stop()
def stop(self, restart=False):
from . import updater_thread
updater_thread.stop()
from . import calibre_db
calibre_db.stop()
log.info("webserver stop (restart=%s)", restart)
self.restart = restart
if self.wsgiserver:
if _GEVENT:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback_from_signal(self.wsgiserver.stop)
def main(*test_args): # runs as standalone on gevent server, main launch script
from gevent.pywsgi import WSGIServer
from galera_node_health import GaleraNodeHealth, examples
import argparse
import configparser
import sys
import traceback
arg_parser = argparse.ArgumentParser(description='Launch galera-node-health using gevent server')
arg_parser.add_argument('-v', dest='print_config', action='store_true', default=False,
help="Print config read during startup")
arg_parser.add_argument('--print-example-config', dest='example_config', action='store_true', default=False,
help="Print example config and exit")
arg_parser.add_argument('-f', dest='config_file', help="Path to file containing config")
arg_parser.add_argument('--docker-logs-fix', dest="docker_logs_fix", action="store_true", default=False,
help=argparse.SUPPRESS)
if not test_args:
args_parsed = arg_parser.parse_args()
else:
args_parsed = arg_parser.parse_args(test_args)
# Fix for `docker logs` command - turn line buffering and redirect to stdout and stderr
if args_parsed.docker_logs_fix:
sys.stdout = open('/dev/stdout', 'w', 1)
sys.stderr = open('/dev/stderr', 'w', 1)
if args_parsed.example_config:
examples.print_config()
if not test_args:
sys.exit(0)
else:
return 'Printed'
if args_parsed.config_file:
config = configparser.ConfigParser(inline_comment_prefixes="#")
try:
with open(args_parsed.config_file, 'r') as f:
config.read_file(f)
checker_port = int(config.get('checker', 'port', fallback=8080))
checker_address = config.get('checker', 'address', fallback='0.0.0.0')
galera_health = GaleraNodeHealth(config_file=args_parsed.config_file, print_config=args_parsed.print_config)
except Exception:
traceback.print_exc()
print("Cannot find or access config file: " + str(args_parsed.config_file))
print("Please make sure it exists and has proper permissions")
sys.exit(1)
else:
checker_port = 8080
checker_address = '0.0.0.0'
galera_health = GaleraNodeHealth(print_config=args_parsed.print_config)
http_server = WSGIServer((checker_address, checker_port), galera_health.flask_app)
try:
http_server.serve_forever()
except OSError as e:
if e.errno == 98:
print("Cannot start server - address already in use: " + checker_address + ":" + str(checker_port))
print("Make sure nothing is running on that address/port or change address/port in config")
sys.exit(1)
else:
raise
except KeyboardInterrupt:
print("Shutting down server because of keyboard interrupt")
finally:
if not http_server.closed:
http_server.close()
