def main():
loglevel = 'INFO'
datefmt = '%b %d %H:%M:%S'
logformat = '%(asctime)s %(levelname)s pysms: %(message)s'
logging.basicConfig(level=loglevel,
stream=sys.stdout,
format=logformat,
datefmt=datefmt)
application = web.application(urls, globals()).wsgifunc()
server = WSGIServer(('', 8088), application)
signal(SIGTERM, exit, arping, server=server)
signal(SIGQUIT, exit, arping, server=server)
signal(SIGINT, exit, arping, server=server)
print "Starting message dispatcher"
dispatcher.start()
print "Starting arping monitor"
arping.start()
print 'Serving on 8088...'
server.start()
while True:
try:
gevent.sleep(0)
except KeyboardInterrupt:
# Switch to main thread, to quit gracefully
gevent.hub.get_hub().switch()
exit(arping, server)
class WebServiceInterface(Interface):
http_port = 80
def __init__(self, *args, **kwargs):
super(WebServiceInterface, self).__init__(*args, **kwargs)
self.wsgi_server = None
def on_start(self):
super(WebServiceInterface, self).on_start()
self.http_socket = create_socket('fd://%s' % self.container.get_shared_socket_fd(self.http_port))
self.wsgi_server = WSGIServer(self.http_socket, Request.application(self.dispatch_request))
self.wsgi_server.start()
def on_stop(self):
self.wsgi_server.stop()
super(WebServiceInterface, self).on_stop()
def dispatch_request(self, request):
trace.set_id()
urls = self.url_map.bind_to_environ(request.environ)
request.urls = urls
try:
endpoint, args = urls.match()
if callable(endpoint):
handler = endpoint(self, request)
response = handler.dispatch(args)
else:
try:
handler = getattr(self, endpoint)
except AttributeError:
raise # FIXME
response = handler(request, **args)
except HTTPException as e:
response = e.get_response(request.environ)
return response
def start_server(input_channels,
cors,
auth_token,
port,
initial_agent,
enable_api=True):
"""Run the agent."""
if enable_api:
app = server.create_app(initial_agent,
cors_origins=cors,
auth_token=auth_token)
else:
app = Flask(__name__)
CORS(app, resources={r"/*": {"origins": cors or ""}})
if input_channels:
rasa_core.channels.channel.register(input_channels,
app,
initial_agent.handle_message,
route="/webhooks/")
if logger.isEnabledFor(logging.DEBUG):
utils.list_routes(app)
http_server = WSGIServer(('0.0.0.0', port), app)
logger.info("Rasa Core server is up and running on "
"{}".format(constants.DEFAULT_SERVER_URL))
http_server.start()
return http_server
class WebService(Service):
"""RPC service with Web server capabilities.
"""
def __init__(self, listen_port, handlers, parameters, shard=0,
custom_logger=None, listen_address=""):
Service.__init__(self, shard, custom_logger)
global logger
from cms.io.GeventLibrary import logger as _logger
logger = _logger
self.__responses = {}
# TODO: why are the following two lines needed?
self._RPCRequestHandler__responses = self.__responses
self._RPCAnswerHandler__responses = self.__responses
handlers += [(r"/rpc_request/([a-zA-Z0-9_-]+)/" \
"([0-9]+)/([a-zA-Z0-9_-]+)",
RPCRequestHandler),
(r"/rpc_answer", RPCAnswerHandler),
(r"/sync_rpc_request/([a-zA-Z0-9_-]+)/" \
"([0-9]+)/([a-zA-Z0-9_-]+)",
SyncRPCRequestHandler)]
self.application = tornado.wsgi.WSGIApplication(handlers, **parameters)
self.application.service = self
# is_proxy_used=True means the content of the header X-Real-IP
# is interpreted as the request IP. This means that if we're
# behind a proxy, it can see the real IP the request is coming
# from. But, to use it, we need to be sure we can trust it
# (i.e., if we are not behind a proxy that sets that header,
# we must not use it).
real_application = self.application
if parameters.get('is_proxy_used', False):
real_application = WSGIXheadersMiddleware(real_application)
self.web_server = WSGIServer((listen_address, listen_port),
real_application)
def run(self):
"""Start the WebService.
Both the WSGI server and the RPC server are started.
"""
self.web_server.start()
Service.run(self)
self.web_server.stop()
@rpc_callback
def _default_callback(self, data, plus, error=None):
"""This is the callback for the RPC method called from a web
page, that just collect the response.
"""
self.__responses[plus] = (data, error)
class RpcAdapterWebSocket(RpcCommAdapter):
def __init__(self,id,ep):
RpcCommAdapter.__init__(self,id,ep)
self.server = None
ep.impl = self
def start(self):
from gevent.pywsgi import WSGIServer
import geventwebsocket
from geventwebsocket.handler import WebSocketHandler
# geventwebsocket.WebSocketServer
# self.server = WSGIServer((self.ep.host,self.ep.port), self._service, handler_class=geventwebsocket.WebSocketHandler)
if self.ep.ssl:
self.server = WSGIServer((self.ep.host,self.ep.port), self._service, handler_class=WebSocketHandler,keyfile=self.ep.keyfile,certfile=self.ep.certfile)
else:
self.server = WSGIServer((self.ep.host,self.ep.port), self._service, handler_class=WebSocketHandler)
print 'websocket server started!'
self.server.start() #.serve_forever()
def stop(self):
self.server.stop()
self.stopmtx.set()
def _http_handler(environ, start_response):
import geventwebsocket
if environ["PATH_INFO"].strip("/") == "version":
start_response("200 OK", [])
agent = "gevent-websocket/%s" % (geventwebsocket.get_version())
return [agent]
else:
start_response("400 Bad Request", [])
return ["WebSocket connection is expected here."]
def _service(self,environ, start_response):
from communicator import RpcCommunicator
print ' new client websocket come in :'#,str(address)
sock = environ.get("wsgi.websocket")
if sock is None:
return self._http_handler(environ, start_response)
conn = RpcConnectionWebSocket(self,self.ep,sock)
self.addConnection(conn)
server = RpcCommunicator.instance().currentServer()
if server.getPropertyValue('userid_check','false') == 'false':
conn.setUserId( str(self.generateSeq()) )
conn.recv()
self.removeConnection(conn)
def sendMessage(self,m):
RpcCommAdapter.sendMessage(self,m)
def init_ws(context):
"""Initialize websocket connection handler.
"""
addr = config.get_value("websocket_address")
logging.info("Opening websocket server on %s", addr)
# TODO: Believe it or not this is not a valid way to check an address
_ws_server = WSGIServer(addr,
WebSocketClientFactory(context),
handler_class=WebSocketHandler)
_ws_server.start()
def preview(filename, options={}):
path = os.path.abspath(filename)
dirpath = os.path.dirname(filename)
realpath = os.path.realpath(path)
if not os.path.isfile(realpath):
error("%s is not a regular file" % filename)
app = Bottle()
@app.route('/')
@view('index')
def index():
return { 'css': options.get('css', True) }
@app.route('/content')
def content():
with open(path, 'rt') as fp:
return render(fp.read(), path)
@app.route('/update-event')
def update():
response.set_header('Content-Type', 'text/event-stream')
response.set_header('Cache-Control', 'no-cache')
mtime = lambda: os.stat(path).st_mtime
last_mtime = mtime()
while True:
if not os.path.exists(path):
error("%s no longer exists" % filename)
current_mtime = mtime()
if current_mtime > last_mtime:
last_mtime = current_mtime
ok('%s updated' % filename)
yield 'data: update\n\n'
time.sleep(0.25)
@app.route('/__moo__/<res:path>')
def moo_static(res):
return static_file(res, root=STATIC_ROOT)
@app.route('/<res:path>')
def static(res):
return static_file(res, root=dirpath)
server = WSGIServer(("127.0.0.1", options.get('port', 0)), app, log=None)
server.start()
url = 'http://127.0.0.1:%d' % server.server_port
info('Server listening at %s. Press Ctrl-C to stop.' % url)
webbrowser.open(url)
server.serve_forever()
def run_server(auction, mapping_expire_time, logger, timezone='Europe/Kiev'):
app.config.update(auction.worker_defaults)
# Replace Flask custom logger
app.logger_name = logger.name
app._logger = logger
app.config['auction'] = auction
app.config['timezone'] = tz(timezone)
app.config['SESSION_COOKIE_PATH'] = '/tenders/{}'.format(auction.auction_doc_id)
app.config['SESSION_COOKIE_NAME'] = 'auction_session'
app.oauth = OAuth(app)
app.remote_oauth = app.oauth.remote_app(
'remote',
consumer_key=app.config['OAUTH_CLIENT_ID'],
consumer_secret=app.config['OAUTH_CLIENT_SECRET'],
request_token_params={'scope': 'email'},
base_url=app.config['OAUTH_BASE_URL'],
access_token_url=app.config['OAUTH_ACCESS_TOKEN_URL'],
authorize_url=app.config['OAUTH_AUTHORIZE_URL']
)
@app.remote_oauth.tokengetter
def get_oauth_token():
return session.get('remote_oauth')
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = 'true'
# Start server on unused port
lisener = get_lisener(auction.worker_defaults["STARTS_PORT"],
host=auction.worker_defaults.get("WORKER_BIND_IP", ""))
app.logger.info(
"Start server on {0}:{1}".format(*lisener.getsockname()),
extra={"JOURNAL_REQUEST_ID": auction.request_id}
)
server = WSGIServer(lisener, app,
log=_LoggerStream(logger),
handler_class=AuctionsWSGIHandler)
server.start()
# Set mapping
mapping_value = "http://{0}:{1}/".format(*lisener.getsockname())
create_mapping(auction.worker_defaults["REDIS_URL"],
auction.auction_doc_id,
mapping_value)
app.logger.info("Server mapping: {} -> {}".format(
auction.auction_doc_id,
mapping_value,
mapping_expire_time
), extra={"JOURNAL_REQUEST_ID": auction.request_id})
# Spawn events functionality
spawn(push_timestamps_events, app,)
spawn(check_clients, app, )
return server
def start_server():
studio.debug = True
# setup secret key
studio.config['SECRET_KEY'] = secret_key = os.urandom(33)
secret_key = base64.b64encode(secret_key).decode('ascii')
# setup OS-assigned port
server = WSGIServer(('localhost', 0), studio, log=sys.stdout,
error_log=sys.stderr)
server.start()
# send key and port to parent process
sys.stdout.write("%d %s" % (server.server_port, secret_key))
sys.stdout.flush() # (needed when output is in a pipe)
# never stop not stopping
server.serve_forever()
def start_serving_files(cachedir, port):
from gevent.pywsgi import WSGIServer
from bottle import route, static_file, default_app
cachedir = os.path.abspath(cachedir)
@route('/cache/:filename#.*#')
def server_static(filename):
response = static_file(filename, root=cachedir, mimetype="application/octet-stream")
if filename.endswith(".rl"):
response.headers["Content-Disposition"] = "inline; filename=collection.pdf"
return response
s = WSGIServer(("", port), default_app())
s.start()
return s
def handle(self, *args, **options):
try:
host = port = None
if len(args) > 0:
host = args[0]
if ':' in host:
host, port = host.rsplit(':')
if host is None:
host = '0.0.0.0'
if port is None:
port = 80
# Monkey Patch for Gevent
monkey.patch_all()
if not options['disable_psycogreen']:
# Monkey Patch for Psycopg2 using psycogreen
import psycogreen.gevent
psycogreen.gevent.patch_psycopg()
application = bootstrap.get_wsgi_application()
if options['disable_socketio']:
from gevent.pywsgi import WSGIServer
server = WSGIServer((host, int(port)), application)
else:
from socketio.server import SocketIOServer
server = SocketIOServer((host, int(port)), application, resource="socket.io")
print 'Starting server on {host}:{port} with {workers} workers.'.format(
host=host,
port=port,
workers=options['child_workers']
)
server.start()
print 'Listening on http://{host}:{port}'.format(host=host, port=port)
for i in range(options['child_workers']):
pid = fork()
if pid == 0:
break
server.serve_forever()
except Exception, ex:
raise CommandError("Exception occurred during gevent wsgi process startup.", ex)
def serve_application(app, serve_forever=True, get_next_message=None):
http_server = WSGIServer(('0.0.0.0', DEFAULT_SERVER_PORT), app)
logger.info("Rasa Core server is up and running on "
"{}".format(DEFAULT_SERVER_URL))
http_server.start()
endpoint = EndpointConfig(url=DEFAULT_SERVER_URL)
start_online_learning_io(endpoint, http_server.stop, get_next_message)
if serve_forever:
try:
http_server.serve_forever()
except Exception as exc:
logger.exception(exc)
return http_server
class HttpServer(Actor):
def pre_start(self, address, responders, default_content_type='text/html'):
self.responders = responders
self.default_content_type = default_content_type
self.server = WSGIServer(address, self.handle_wsgi_request)
self.server.start()
def post_stop(self):
self.server.stop()
def handle_wsgi_request(self, env, start_response):
ch = Channel()
req = Request(ch, env, start_response, content_type=self.default_content_type)
self << ('handle', req)
return response_stream(ch)
def receive(self, msg):
if ('handle', ANY) == msg:
_, req = msg
try:
responder = self.get_responder(req.env['PATH_INFO'])
if responder:
responder, args, kwargs = responder
self.spawn(RequestHandler.using(req, responder, args, kwargs))
else:
req.start_response('404 Not Found', [('Content-Type', 'text/html')])
req.write('<h1>404 Not Found</h1>\n')
req.write('The page you tried to reach could not be found.\n')
req.close()
except:
_send_500(req)
raise
elif 'get-addr' == msg:
self.reply(self.server.address)
else:
raise Unhandled
def get_responder(self, path):
for pattern, responder in self.responders:
m = re.compile(pattern).match(path)
if m:
args = m.groups()
kwargs = m.groupdict()
return responder, args if not kwargs else (), kwargs
class WebService(Service):
"""RPC service with Web server capabilities.
"""
def __init__(self, listen_port, handlers, parameters, shard=0,
listen_address=""):
super(WebService, self).__init__(shard)
static_files = parameters.pop('static_files', [])
rpc_enabled = parameters.pop('rpc_enabled', False)
is_proxy_used = parameters.pop('is_proxy_used', False)
self.wsgi_app = tornado.wsgi.WSGIApplication(handlers, **parameters)
self.wsgi_app.service = self
for entry in static_files:
self.wsgi_app = SharedDataMiddleware(
self.wsgi_app, {"/static": entry})
if rpc_enabled:
self.wsgi_app = DispatcherMiddleware(
self.wsgi_app, {"/rpc": RPCMiddleware(self)})
# If is_proxy_used is set to True we'll use the content of the
# X-Forwarded-For HTTP header (if provided) to determine the
# client IP address, ignoring the one the request came from.
# This allows to use the IP lock behind a proxy. Activate it
# only if all requests come from a trusted source (if clients
# were allowed to directlty communicate with the server they
# could fake their IP and compromise the security of IP lock).
if is_proxy_used:
self.wsgi_app = ProxyFix(self.wsgi_app)
self.web_server = WSGIServer((listen_address, listen_port),
self.wsgi_app)
def run(self):
"""Start the WebService.
Both the WSGI server and the RPC server are started.
"""
self.web_server.start()
Service.run(self)
self.web_server.stop()
class Web(object):
"""
Dissonance's WSGI server. Routes requests to their appropriate module. See `dissonance.module.Module.app`
for more details.
"""
_url_map = Map([
Rule('/<module>/', endpoint='module', defaults={'rest': ''}),
Rule('/<module>/<path:rest>', endpoint='module')
])
def __init__(self, client, opts):
self._client = client
self._opts = opts
self._bind_addr = self._opts['listen_host'], int(self._opts['listen_port'])
self._server = WSGIServer(self._bind_addr, self._wsgi_app)
def _wsgi_app(self, environ, start_response):
urls = self._url_map.bind_to_environ(environ)
try:
endpoint, args = urls.match()
handler = getattr(self, '_handle_%s' % endpoint)
return handler(args, environ, start_response)
except HTTPException as e:
return e(environ, start_response)
def _handle_module(self, args, environ, start_response):
module = self._client.modules.get_module(args['module'])
if module and module.is_web:
original_script_name = environ.get('SCRIPT_NAME', '')
environ['SCRIPT_NAME'] = original_script_name + '/' + args['module']
environ['PATH_INFO'] = args['rest']
return module.app(environ, start_response)
return NotFound()(environ, start_response)
def start(self):
logger.info("Starting web server on %s:%s", *self._bind_addr)
self._server.start()
def stop(self):
logger.info("Stopping web server on %s:%s", *self._bind_addr)
self._server.stop()
class Frontend(object):
def start(self, start_wsgi_server=False, forever=False):
"""
Start the frontend's greenlets.
If *start_wsgi_server* is True, also create the frontend's main WSGI
application and serve it with gevent's WSGI server.
"""
# Start the frontend WSGI server if requested
if start_wsgi_server:
wsgi_app = self.create_wsgi_app(
{
"pyramid.debug_authorization": "true",
"pyramid.debug_routematch": "true",
"pyramid.reload_templates": "true",
"pyramid.debug_notfound": "true",
}
)
logfile = open("/tmp/jarvis.log", "w")
self.wsgi_server = WSGIServer(("127.0.0.1", 9017), wsgi_app, log=logfile)
if forever:
self.wsgi_server.serve_forever()
else:
self.wsgi_server.start()
else:
self.wsgi_server = None
def stop(self):
if self.wsgi_server is not None:
self.wsgi_server.stop()
def create_wsgi_app(self, settings):
# Create applincation
config = Configurator(settings=settings)
config.add_route("state_update", "/state/update/{id}/")
config.add_route("state_stream_json", "/state/stream/json/{session_id}/")
config.add_route("state_stream_eventsource", "/state/stream/eventsource/{session_id}/")
static_path = op.join(op.dirname(__file__), "media")
config.add_static_view(name="static", path=static_path)
config.scan()
return config.make_wsgi_app()
class ContractingUserTestCase(unittest.TestCase):
""""""
def setUp(self):
#self._testMethodName
self.app = Bottle()
setup_routing(self.app)
self.server = WSGIServer(('localhost', 20602), self.app, log=None)
self.server.start()
self.client = ContractingClient(API_KEY, host_url=HOST_URL,
api_version=API_VERSION)
with open(ROOT + 'contract_' + TEST_CONTRACT_KEYS.contract_id + '.json') as contract:
self.contract = munchify(load(contract))
self.contract.update({'access':{"token": API_KEY}})
def tearDown(self):
self.server.stop()
###########################################################################
# CREATE ITEM TEST
###########################################################################
def test_create_contract(self):
setup_routing(self.app, routs=["contract_create"])
contract = munchify({'data': 'contract'})
self.client.create_contract(contract)
###########################################################################
# DOCUMENTS FILE TEST
###########################################################################
def test_upload_contract_document(self):
setup_routing(self.app, routs=["contract_document_create"])
file_ = StringIO()
file_.name = 'test_document.txt'
file_.write("test upload contract document text data")
file_.seek(0)
doc = self.client.upload_document(file_, self.contract)
self.assertEqual(doc.data.title, file_.name)
self.assertEqual(doc.data.id, TEST_CONTRACT_KEYS.new_document_id)
file_.close()
def start_server(manager, host='0.0.0.0', port=8080):
"""
Start the server on http://{host}:{port} and return
"""
# TODO: Ewwww, globals :(
global MANAGER
MANAGER = manager
# Construct & start the server
server = WSGIServer(
(host, port),
app,
handler_class=WebSocketHandler
)
print 'NOTE: You must have a websocket display setup for display in browser to work!'
print 'Starting server on {host}:{port}...'.format(
host=host,
port=port
)
server.start()
class HTTPProxyServer(object):
def __init__(self, ip, port, app, log='default'):
self.ip = ip
self.port = port
self.app = app
self.server = WSGIServer((self.ip, self.port), log=log,
application=self.app.application, spawn=Pool(500), handler_class=ProxyHandler)
def start(self):
self.server.start()
def run(self):
self.server.serve_forever()
def stop(self):
self.server.stop()
@property
def closed(self):
return self.server.closed
def main():
app.logger.info("setting context")
context = zmq.Context()
gevent.spawn(zmq_qry_pub, context)
# websocket server: copies inproc zmq messages to websocket
ws_server = WSGIServer(
('', 9999),
WebSocketApp(context),
handler_class=WebSocketHandler
)
http_server = WSGIServer(('', 8080), app)
http_server.start()
ws_server.start()
zmq_sub(context)
class ViewerTenderTestCase(unittest.TestCase):
""""""
def setUp(self):
#self._testMethodName
self.app = Bottle()
setup_routing(self.app)
self.server = WSGIServer(('localhost', 20602), self.app, log=None)
self.server.start()
self.client = tender_client.TendersClient('', host_url=HOST_URL, api_version=API_VERSION)
with open(ROOT + 'tenders.json') as tenders:
self.tenders = munchify(load(tenders))
with open(ROOT + TEST_KEYS.tender_id + '.json') as tender:
self.tender = munchify(load(tender))
def tearDown(self):
self.server.stop()
def test_get_tenders(self):
setup_routing(self.app, routs=["tenders"])
tenders = self.client.get_tenders()
self.assertIsInstance(tenders, Iterable)
self.assertEqual(tenders, self.tenders.data)
def test_get_tender(self):
setup_routing(self.app, routs=["tender"])
tender = self.client.get_tender(TEST_KEYS.tender_id)
self.assertEqual(tender, self.tender)
def test_get_tender_location_error(self):
setup_routing(self.app, routs=["tender"])
tender = self.client.get_tender(TEST_KEYS.error_id)
self.assertEqual(tender, munchify(loads(location_error('tender'))))
def test_offset_error(self):
setup_routing(self.app, routs=['offset_error'])
tenders = self.client.get_tenders()
self.assertIsInstance(tenders, Iterable)
self.assertEqual(tenders, self.tenders.data)
class ViewerPlanTestCase(unittest.TestCase):
""""""
def setUp(self):
self.app = Bottle()
setup_routing(self.app)
self.server = WSGIServer(('localhost', 20602), self.app, log=None)
self.server.start()
self.client = plan_client.PlansClient('', host_url=HOST_URL, api_version=API_VERSION)
with open(ROOT + 'plans.json') as plans:
self.plans = munchify(load(plans))
with open(ROOT + 'plan_' + TEST_PLAN_KEYS.plan_id + '.json') as plan:
self.plan = munchify(load(plan))
def tearDown(self):
self.server.stop()
def test_get_plans(self):
setup_routing(self.app, routs=["plans"])
plans = self.client.get_plans()
self.assertIsInstance(plans, Iterable)
self.assertEqual(plans, self.plans.data)
def test_get_plan(self):
setup_routing(self.app, routs=["plan"])
plan = self.client.get_plan(TEST_PLAN_KEYS.plan_id)
self.assertEqual(plan, self.plan)
def test_get_plan_location_error(self):
setup_routing(self.app, routs=["plan"])
tender = self.client.get_plan(TEST_PLAN_KEYS.error_id)
self.assertEqual(tender, munchify(loads(location_error('plan'))))
def test_offset_error(self):
setup_routing(self.app, routs=['plan_offset_error'])
plans = self.client.get_plans()
self.assertIsInstance(plans, Iterable)
self.assertEqual(plans, self.plans.data)
def handle_channels(self, channels,
http_port=constants.DEFAULT_SERVER_PORT,
serve_forever=True):
# type: (List[InputChannel], int, bool) -> WSGIServer
"""Start a webserver attaching the input channels and handling msgs.
If ``serve_forever`` is set to ``True``, this call will be blocking.
Otherwise the webserver will be started, and the method will
return afterwards."""
from flask import Flask
app = Flask(__name__)
rasa_core.channels.channel.register(channels,
app,
self.handle_message,
route="/webhooks/")
http_server = WSGIServer(('0.0.0.0', http_port), app)
http_server.start()
if serve_forever:
http_server.serve_forever()
return http_server
class WebService(Service):
"""RPC service with Web server capabilities.
"""
def __init__(self, listen_port, handlers, parameters, shard=0,
listen_address=""):
super().__init__(shard)
static_files = parameters.pop('static_files', [])
rpc_enabled = parameters.pop('rpc_enabled', False)
rpc_auth = parameters.pop('rpc_auth', None)
auth_middleware = parameters.pop('auth_middleware', None)
is_proxy_used = parameters.pop('is_proxy_used', None)
num_proxies_used = parameters.pop('num_proxies_used', None)
self.wsgi_app = tornado.wsgi.WSGIApplication(handlers, **parameters)
self.wsgi_app.service = self
for entry in static_files:
# TODO If we will introduce a flag to trigger autoreload in
# Jinja2 templates, use it to disable the cache arg here.
self.wsgi_app = SharedDataMiddleware(
self.wsgi_app, {"/static": entry},
cache=True, cache_timeout=SECONDS_IN_A_YEAR,
fallback_mimetype="application/octet-stream")
self.file_cacher = FileCacher(self)
self.wsgi_app = FileServerMiddleware(self.file_cacher, self.wsgi_app)
if rpc_enabled:
self.wsgi_app = DispatcherMiddleware(
self.wsgi_app, {"/rpc": RPCMiddleware(self, rpc_auth)})
# The authentication middleware needs to be applied before the
# ProxyFix as otherwise the remote address it gets is the one
# of the proxy.
if auth_middleware is not None:
self.wsgi_app = auth_middleware(self.wsgi_app)
self.auth_handler = self.wsgi_app
# If we are behind one or more proxies, we'll use the content
# of the X-Forwarded-For HTTP header (if provided) to determine
# the client IP address, ignoring the one the request came from.
# This allows to use the IP lock behind a proxy. Activate it
# only if all requests come from a trusted source (if clients
# were allowed to directlty communicate with the server they
# could fake their IP and compromise the security of IP lock).
if num_proxies_used is None:
if is_proxy_used:
num_proxies_used = 1
else:
num_proxies_used = 0
if num_proxies_used > 0:
self.wsgi_app = ProxyFix(self.wsgi_app, num_proxies_used)
self.web_server = WSGIServer((listen_address, listen_port), self)
def __call__(self, environ, start_response):
"""Execute this instance as a WSGI application.
See the PEP for the meaning of parameters. The separation of
__call__ and wsgi_app eases the insertion of middlewares.
"""
return self.wsgi_app(environ, start_response)
def run(self):
"""Start the WebService.
Both the WSGI server and the RPC server are started.
"""
self.web_server.start()
Service.run(self)
self.web_server.stop()
class HttpServer(threading.Thread):
daemon = True # causes server to stop on main thread quitting
def __init__(self,
api,
port=0,
host='0.0.0.0',
api_args=None,
api_kwargs=None,
ssl=None):
self.api_args = api_args
self.api_kwargs = api_kwargs
if self.api_args is None:
self.api_args = []
if self.api_kwargs is None:
self.api_kwargs = {}
self.api_class = api
self.port = port
self.host = host
self.started = threading.Event()
self.failed = None
self.ssl = ssl
threading.Thread.__init__(self)
def run(self):
self.api = self.api_class(*self.api_args, **self.api_kwargs)
port = self.port
while True:
if self.port == 0:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('localhost', 0))
self.port = sock.getsockname()[1]
sock.close()
if self.port == 0:
print("ERROR! Port still 0!")
break
try:
# self.api.app.before_first_request = self.__setstarted()
if self.ssl is not None:
self.server = WSGIServer((self.host, self.port),
self.api.app,
handler_class=WebSocketHandler,
**self.ssl)
else:
self.server = WSGIServer((self.host, self.port),
self.api.app,
handler_class=WebSocketHandler)
self.server.start()
self.__setstarted()
self.server.serve_forever()
except socket.error as e:
if e.errno == 48:
if port != 0:
raise
else:
self.port = port
continue
else:
self.failed = e
self.__setstarted()
break
def stop(self):
self.api.stop()
def __setstarted(self):
self.api.port = self.port
self.started.set()
if self.failed is None:
print("HttpServer running")
else:
print("HttpServer failed to start")
class server:
wsgiserver = None
restart = False
def __init__(self):
signal.signal(signal.SIGINT, self.killServer)
signal.signal(signal.SIGTERM, self.killServer)
def start_gevent(self):
try:
ssl_args = dict()
certfile_path = web.ub.config.get_config_certfile()
keyfile_path = web.ub.config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(
keyfile_path):
ssl_args = {
"certfile": certfile_path,
"keyfile": keyfile_path
}
else:
web.app.logger.info(
'The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s'
% (certfile_path, keyfile_path))
if os.name == 'nt':
self.wsgiserver = WSGIServer(
('0.0.0.0', web.ub.config.config_port),
web.app,
spawn=Pool(),
**ssl_args)
else:
self.wsgiserver = WSGIServer(('', web.ub.config.config_port),
web.app,
spawn=Pool(),
**ssl_args)
web.py3_gevent_link = self.wsgiserver
self.wsgiserver.serve_forever()
except SocketError:
try:
web.app.logger.info(
'Unable to listen on \'\', trying on IPv4 only...')
self.wsgiserver = WSGIServer(
('0.0.0.0', web.ub.config.config_port),
web.app,
spawn=Pool(),
**ssl_args)
web.py3_gevent_link = self.wsgiserver
self.wsgiserver.serve_forever()
except (OSError, SocketError) as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
except Exception:
web.app.logger.info("Unknown error while starting gevent")
def startServer(self):
if gevent_present:
web.app.logger.info('Starting Gevent server')
# leave subprocess out to allow forking for fetchers and processors
self.start_gevent()
else:
try:
ssl = None
web.app.logger.info('Starting Tornado server')
certfile_path = web.ub.config.get_config_certfile()
keyfile_path = web.ub.config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(
keyfile_path):
ssl = {
"certfile": certfile_path,
"keyfile": keyfile_path
}
else:
web.app.logger.info(
'The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s'
% (certfile_path, keyfile_path))
# Max Buffersize set to 200MB
http_server = HTTPServer(WSGIContainer(web.app),
max_buffer_size=209700000,
ssl_options=ssl)
http_server.listen(web.ub.config.config_port)
self.wsgiserver = IOLoop.instance()
self.wsgiserver.start()
# wait for stop signal
self.wsgiserver.close(True)
except SocketError as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
# ToDo: Somehow caused by circular import under python3 refactor
if sys.version_info > (3, 0):
self.restart = web.py3_restart_Typ
if self.restart == True:
web.app.logger.info("Performing restart of Calibre-Web")
web.helper.global_WorkerThread.stop()
if os.name == 'nt':
arguments = ["\"" + sys.executable + "\""]
for e in sys.argv:
arguments.append("\"" + e + "\"")
os.execv(sys.executable, arguments)
else:
os.execl(sys.executable, sys.executable, *sys.argv)
else:
web.app.logger.info("Performing shutdown of Calibre-Web")
web.helper.global_WorkerThread.stop()
sys.exit(0)
def setRestartTyp(self, starttyp):
self.restart = starttyp
# ToDo: Somehow caused by circular import under python3 refactor
web.py3_restart_Typ = starttyp
def killServer(self, signum, frame):
self.stopServer()
def stopServer(self):
# ToDo: Somehow caused by circular import under python3 refactor
if sys.version_info > (3, 0):
if not self.wsgiserver:
if gevent_present:
self.wsgiserver = web.py3_gevent_link
else:
self.wsgiserver = IOLoop.instance()
if self.wsgiserver:
if gevent_present:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback(self.wsgiserver.stop)
@staticmethod
def getNameVersion():
if gevent_present:
return {'Gevent': 'v' + geventVersion}
else:
return {'Tornado': 'v' + tornadoVersion}
def run(debug, no_browser, m, mu, mc, f):
"""
Sacredboard.
\b
Sacredboard is a monitoring dashboard for Sacred.
Homepage: http://github.com/chovanecm/sacredboard
Example usage:
\b
sacredboard -m sacred
Starts Sacredboard on default port (5000) and connects to
a local MongoDB database called 'sacred'. Opens web browser.
Note: MongoDB must be listening on localhost.
\b
sacredboard -m 192.168.1.1:27017:sacred
Starts Sacredboard on default port (5000) and connects to
a MongoDB database running on 192.168.1.1 on port 27017
to a database called 'sacred'. Opens web browser.
\b
sacredboard -mu mongodb://user:pwd@host/admin?authMechanism=SCRAM-SHA-1 sacred
Starts Sacredboard on default port (5000) and connects to
a MongoDB database running on localhost on port 27017
to a database called 'sacred'. Opens web browser.
\b
sacredboard -m sacred -mc default.runs
Starts Sacredboard on default port (5000) and connects to
a local MongoDB database called 'sacred' and uses the Sacred's 0.6
default collection 'default.runs' to search the runs in.
Opens web browser.
Note: MongoDB must be listening on localhost.
"""
if m or mu != (None, None):
add_mongo_config(app, m, mu, mc)
app.config["data"].connect()
elif f:
app.config["data"] = FileStorage(f)
else:
print("Must specify either a mongodb instance or " +
"a path to a file storage.\nRun sacredboard --help "
"for more information.",
file=sys.stderr)
sys.exit(1)
app.config['DEBUG'] = debug
app.debug = debug
jinja_filters.setup_filters(app)
routes.setup_routes(app)
metrics.initialize(app)
if debug:
app.run(host="0.0.0.0", debug=True)
else:
for port in range(5000, 5050):
http_server = WSGIServer(('0.0.0.0', port), app)
try:
http_server.start()
except OSError as e:
# try next port
continue
print("Starting sacredboard on port %d" % port)
if not no_browser:
click.launch("http://127.0.0.1:%d" % port)
http_server.serve_forever()
break
class server:
wsgiserver = None
restart= False
def __init__(self):
signal.signal(signal.SIGINT, self.killServer)
signal.signal(signal.SIGTERM, self.killServer)
def start_gevent(self):
try:
ssl_args = dict()
if web.ub.config.get_config_certfile() and web.ub.config.get_config_keyfile():
ssl_args = {"certfile": web.ub.config.get_config_certfile(),
"keyfile": web.ub.config.get_config_keyfile()}
if os.name == 'nt':
self.wsgiserver= WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
else:
self.wsgiserver = WSGIServer(('', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
self.wsgiserver.serve_forever()
except SocketError:
try:
web.app.logger.info('Unable to listen on \'\', trying on IPv4 only...')
self.wsgiserver = WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
self.wsgiserver.serve_forever()
except (OSError, SocketError) as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
except Exception:
web.app.logger.info("Unknown error while starting gevent")
def startServer(self):
if gevent_present:
web.app.logger.info('Starting Gevent server')
# leave subprocess out to allow forking for fetchers and processors
self.start_gevent()
else:
try:
web.app.logger.info('Starting Tornado server')
if web.ub.config.get_config_certfile() and web.ub.config.get_config_keyfile():
ssl={"certfile": web.ub.config.get_config_certfile(),
"keyfile": web.ub.config.get_config_keyfile()}
else:
ssl=None
# Max Buffersize set to 200MB
http_server = HTTPServer(WSGIContainer(web.app),
max_buffer_size = 209700000,
ssl_options=ssl)
http_server.listen(web.ub.config.config_port)
self.wsgiserver=IOLoop.instance()
self.wsgiserver.start()
# wait for stop signal
self.wsgiserver.close(True)
except SocketError as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
if self.restart == True:
web.app.logger.info("Performing restart of Calibre-Web")
web.helper.global_WorkerThread.stop()
if os.name == 'nt':
arguments = ["\"" + sys.executable + "\""]
for e in sys.argv:
arguments.append("\"" + e + "\"")
os.execv(sys.executable, arguments)
else:
os.execl(sys.executable, sys.executable, *sys.argv)
else:
web.app.logger.info("Performing shutdown of Calibre-Web")
web.helper.global_WorkerThread.stop()
sys.exit(0)
def setRestartTyp(self,starttyp):
self.restart=starttyp
def killServer(self, signum, frame):
self.stopServer()
def stopServer(self):
if gevent_present:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback(self.wsgiserver.stop)
@staticmethod
def getNameVersion():
if gevent_present:
return {'Gevent':'v'+geventVersion}
else:
return {'Tornado':'v'+tornadoVersion}
class WebInternalReader(object):
def __init__(self):
self._log = logging.getLogger("WebInternalReader")
memcached_client = memcache.Client(_memcached_nodes)
self._central_connection = get_central_connection()
self._cluster_row = get_cluster_row(self._central_connection)
self._node_local_connection = get_node_local_connection()
self._deliverator = Deliverator()
self._zeromq_context = zmq.Context()
self._pull_server = GreenletPULLServer(
self._zeromq_context, _web_internal_reader_pipeline_address,
self._deliverator)
self._pull_server.link_exception(self._unhandled_greenlet_exception)
self._data_reader_clients = list()
self._data_readers = list()
for node_name, address in zip(_node_names, _data_reader_addresses):
resilient_client = GreenletResilientClient(
self._zeromq_context,
node_name,
address,
_client_tag,
_web_internal_reader_pipeline_address,
self._deliverator,
connect_messages=[])
resilient_client.link_exception(self._unhandled_greenlet_exception)
self._data_reader_clients.append(resilient_client)
data_reader = DataReader(node_name, resilient_client)
self._data_readers.append(data_reader)
self._space_accounting_dealer_client = GreenletDealerClient(
self._zeromq_context, _local_node_name,
_space_accounting_server_address)
self._space_accounting_dealer_client.link_exception(
self._unhandled_greenlet_exception)
push_client = GreenletPUSHClient(
self._zeromq_context,
_local_node_name,
_space_accounting_pipeline_address,
)
self._accounting_client = SpaceAccountingClient(
_local_node_name, self._space_accounting_dealer_client,
push_client)
self._event_push_client = EventPushClient(self._zeromq_context,
"web-internal-reader")
# message sent to data readers telling them the server
# is (re)starting, thereby invalidating any archvies or retrieved
# that are in progress for this node
timestamp = create_timestamp()
self._event_push_client.info("web-reader-start",
"web reader (re)start",
timestamp_repr=repr(timestamp),
source_node_name=_local_node_name)
self._watcher = Watcher(_stats, self._data_reader_clients,
self._event_push_client)
self.application = Application(memcached_client,
self._central_connection,
self._node_local_connection,
self._cluster_row, self._data_readers,
self._accounting_client,
self._event_push_client, _stats)
self.wsgi_server = WSGIServer(
(_web_internal_reader_host, _web_internal_reader_port),
application=self.application,
backlog=_wsgi_backlog)
def start(self):
self._space_accounting_dealer_client.start()
self._pull_server.start()
self._watcher.start()
for client in self._data_reader_clients:
client.start()
self.wsgi_server.start()
def stop(self):
self._log.info("stopping wsgi web server")
self.wsgi_server.stop()
self._accounting_client.close()
self._log.debug("killing greenlets")
self._space_accounting_dealer_client.kill()
self._pull_server.kill()
self._watcher.kill()
for client in self._data_reader_clients:
client.kill()
self._log.debug("joining greenlets")
self._space_accounting_dealer_client.join()
self._pull_server.join()
self._watcher.join()
for client in self._data_reader_clients:
client.join()
self._log.debug("closing zmq")
self._event_push_client.close()
self._zeromq_context.term()
self._log.info("closing database connections")
self._central_connection.close()
self._node_local_connection.close()
def _unhandled_greenlet_exception(self, greenlet_object):
try:
greenlet_object.get()
except Exception:
self._log.exception(str(greenlet_object))
class OrionContextBroker(ResourceManagementXAE):
def __init__(self,
orion_host="http://localhost:1026",
orion_api="v2",
labels=None,
accumulate_address=None,
*args,
**kw):
super(OrionContextBroker, self).__init__(*args, **kw)
if isinstance(labels, str):
self.labels = {labels}
elif hasattr(labels, '__iter__'):
self.labels = set(labels)
elif labels is None:
self.labels = ["openmtc:sensor_data"]
else:
self.labels = None
self._entity_names = {}
self._subscription_endpoints = {}
self._subscription_services = {}
# accumulate address
if not accumulate_address:
accumulate_address = "http://" + self._get_auto_host(
orion_host) + ":8080"
# Orion API
self._dry_run = not orion_host
self.orion_api = OrionAPI(
orion_host=orion_host,
api_version=orion_api,
accumulate_endpoint="{}/accumulate".format(accumulate_address))
if not self._dry_run:
self._dry_run = not self.orion_api.is_host_alive()
# Subscription Sink for OCB
self.app = Flask(__name__)
self.app.add_url_rule('/accumulate',
'process_notification',
self.process_notification,
methods=["POST"])
accumulate_ip, accumulate_port = urlparse(
accumulate_address).netloc.rsplit(':', 1)
self.server = WSGIServer(("0.0.0.0", int(accumulate_port)), self.app)
self.server.start()
@staticmethod
def _get_auto_host(ep):
try:
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
netloc = urlparse(ep).netloc.split(':')
s.connect((netloc[0], int(netloc[1])))
host = s.getsockname()[0]
s.close()
except:
host = "127.0.0.1"
return host
def process_notification(self):
self.logger.debug("Got from Subscription {}".format(request.json))
try:
actuator = self.get_resource(
self._subscription_endpoints[request.json["subscriptionId"]])
except KeyError:
# ignore not deleted old subscriptions
pass
else:
self.push_content(actuator,
request.json["data"][0]["cmd"]["value"])
return Response(status=200, headers={})
def _on_register(self):
self._discover_openmtc_ipe_entities()
def _on_shutdown(self):
for subscription_id, fiware_service in self._subscription_services.items(
):
self.orion_api.unsubscribe(subscription_id, fiware_service)
def _sensor_filter(self, sensor_info):
if self.labels:
return len(self.labels.intersection(
sensor_info['sensor_labels'])) > 0
else:
return True
@staticmethod
def _get_entity_name(sensor_info):
device_type = "sensor" if sensor_info.get("sensor_labels",
None) else "actuator"
try:
id_label = [
x for x in sensor_info['{}_labels'.format(device_type)]
if x.startswith('openmtc:id:')
].pop()
cse_id, dev_id = re.sub('^openmtc:id:', '',
id_label).split('/')[:2]
except (IndexError, ValueError):
cse_id = sensor_info['cse_id']
dev_id = sensor_info['dev_name']
try:
f_s, e_pre = cse_id.split('~', 1)
except ValueError:
f_s = ''
e_pre = cse_id
return re.sub('[\W]', '_', f_s), '%s-%s' % (e_pre, dev_id)
def _sensor_data_cb(self, sensor_info, sensor_data):
if self._dry_run:
return
try:
fiware_service, entity_name = self._entity_names[sensor_info['ID']]
except KeyError:
self._entity_names[sensor_info['ID']] = self._get_entity_name(
sensor_info)
fiware_service, entity_name = self._entity_names[sensor_info['ID']]
self.orion_api.create_entity(entity_name,
fiware_service=fiware_service)
self.orion_api.update_attributes(entity_name,
sensor_data,
fiware_service=fiware_service)
def _new_actuator(self, actuator_info):
try:
fiware_service, entity_name = self._entity_names[
actuator_info['ID']]
except KeyError:
self._entity_names[actuator_info['ID']] = self._get_entity_name(
actuator_info)
fiware_service, entity_name = self._entity_names[
actuator_info['ID']]
self.logger.info("Create new Entity {} on Fiware Service {}".format(
entity_name, fiware_service))
if self._dry_run:
return
self.orion_api.create_entity(entity_name,
fiware_service=fiware_service)
data_dummy = {
'v': "none",
'bn': "none",
'n': "cmd",
'u': "none",
't': "none"
}
self.orion_api.update_attributes(entity_name,
data_dummy,
fiware_service=fiware_service)
subscription_id = self.orion_api.subscribe(
entity_name, fiware_service=fiware_service)
self._subscription_endpoints[subscription_id] = actuator_info['ID']
self._subscription_services[subscription_id] = fiware_service
class PFSApi:
# pylint: disable=too-many-instance-attributes
# Nine is reasonable in this case.
def __init__(
self,
pathfinding_service: PathfindingService,
one_to_n_address: Address,
operator: str,
info_message: str = DEFAULT_INFO_MESSAGE,
service_fee: TokenAmount = TokenAmount(0),
debug_mode: bool = False,
) -> None:
flask_app = Flask(__name__)
self.flask_app = DispatcherMiddleware(
NotFound(),
{
"/metrics": make_wsgi_app(registry=metrics.REGISTRY),
API_PATH: flask_app.wsgi_app,
},
)
self.api = ApiWithErrorHandler(flask_app)
self.rest_server: Optional[WSGIServer] = None
self.one_to_n_address = one_to_n_address
self.pathfinding_service = pathfinding_service
self.service_fee = service_fee
self.operator = operator
self.info_message = info_message
# Enable cross origin requests
@flask_app.after_request
def after_request(response: Response) -> Response: # pylint: disable=unused-variable
response.headers.add("Access-Control-Allow-Origin", "*")
response.headers.add("Access-Control-Allow-Headers",
"Origin, Content-Type, Accept")
response.headers.add("Access-Control-Allow-Methods",
"GET,POST,OPTIONS")
return response
resources: List[Tuple[str, Resource, Dict, str]] = [
(
"/v1/<token_network_address>/paths",
PathsResource,
dict(debug_mode=debug_mode),
"paths",
),
("/v1/<token_network_address>/payment/iou", IOUResource, {},
"payments"),
("/v1/<token_network_address>/feedback", FeedbackResource, {},
"feedback"),
(
"/v1/<token_network_address>/suggest_partner",
SuggestPartnerResource,
{},
"suggest_partner",
),
("/v1/online_addresses", OnlineAddressesResource, {},
"online_addresses"),
("/v1/info", InfoResource, {}, "info"),
("/v2/info", InfoResource2, {}, "info2"),
(
"/v1/address/<checksummed_address>/metadata",
AddressMetadataResource,
{},
"address_metadata",
),
]
if debug_mode:
log.warning(
"The debug REST API is enabled. Don't do this on public nodes."
)
resources.extend([
(
"/v1/_debug/routes/<token_network_address>/<source_address>",
cast(Resource, DebugPathResource),
{},
"debug1",
),
(
"/v1/_debug/routes/<token_network_address>/<source_address>/<target_address>", # noqa
DebugPathResource,
{},
"debug2",
),
("/v1/_debug/ious/<source_address>", DebugIOUResource, {},
"debug3"),
("/v1/_debug/stats", DebugStatsResource, {}, "debug4"),
])
for endpoint_url, resource, kwargs, endpoint in resources:
kwargs.update({
"pathfinding_service": pathfinding_service,
"api": self
})
self.api.add_resource(resource,
endpoint_url,
resource_class_kwargs=kwargs,
endpoint=endpoint)
def run(self, host: str, port: int) -> None:
self.rest_server = WSGIServer((host, port), self.flask_app)
self.rest_server.start()
log.info("Running endpoint", endpoint=f"http://{host}:{port}")
def stop(self) -> None:
if self.rest_server:
self.rest_server.stop()
if env['PATH_INFO'] == '/status/disk_io':
start_response('200 OK', [("Content-Type", "application/json")])
msg = make_perf_list(lambda x: x[1]['disk_io'])
return [msg]
else:
start_response('404 Not Found', [("Content-Type", "application/json")])
msg = json.dumps({"data": "Not Found"})
return [msg]
if __name__ == '__main__':
#TODO: 设置监视greenlet
q = queue.JoinableQueue()
wsgi_server = WSGIServer(('', 23458), application)
wsgi_server.start()
def server_exit():
wsgi_server.stop(0)
gevent.killall(gs, block=False)
gsignal(signal.SIGALRM, signal.SIG_IGN)
gsignal(signal.SIGHUP, signal.SIG_IGN)
gsignal(signal.SIGINT, server_exit)
gsignal(signal.SIGTERM, server_exit)
threading.Thread(target=process.process, args=(thread_queue, )).start()
xen_manager = XenserverManager(q)
gs.append(gevent.spawn(xen_manager.make_10m_perf_url))
gs.append(gevent.spawn(xen_manager.make_2h_perf_url))
gs.append(gevent.spawn(xen_manager.make_1w_perf_url))
class ServiceApi:
def __init__(
self,
pathfinding_service: PathfindingService,
service_fee: TokenAmount = TokenAmount(0),
debug_mode: bool = False,
):
self.flask_app = Flask(__name__)
self.api = ApiWithErrorHandler(self.flask_app)
self.rest_server: WSGIServer = None
self.pathfinding_service = pathfinding_service
self.service_fee = service_fee
resources: List[Tuple[str, Resource, Dict, str]] = [
(
"/<token_network_address>/paths",
PathsResource,
dict(debug_mode=debug_mode),
"paths",
),
("/<token_network_address>/payment/iou", IOUResource, {},
"payments"),
("/info", InfoResource, {}, "info"),
]
if debug_mode:
log.warning(
"The debug REST API is enabled. Don't do this on public nodes."
)
resources.extend([
(
"/_debug/routes/<token_network_address>/<source_address>",
DebugEndpoint,
{},
"debug1",
),
(
"/_debug/routes/<token_network_address>/<source_address>/<target_address>",
DebugEndpoint,
{},
"debug2",
),
("/_debug/ious/<source_address>", DebugEndpointIOU, {},
"debug3"),
])
for endpoint_url, resource, kwargs, endpoint in resources:
endpoint_url = API_PATH + endpoint_url
kwargs.update({
"pathfinding_service": pathfinding_service,
"service_api": self
})
self.api.add_resource(resource,
endpoint_url,
resource_class_kwargs=kwargs,
endpoint=endpoint)
def run(self,
host: str = DEFAULT_API_HOST,
port: int = DEFAULT_API_PORT) -> None:
self.rest_server = WSGIServer((host, port), self.flask_app)
self.rest_server.start()
log.info("Running endpoint", endpoint=f"{host}:{port}")
def stop(self) -> None:
self.rest_server.stop()
class WebService(Service):
"""RPC service with Web server capabilities.
"""
def __init__(self, listen_port, handlers, parameters, shard=0,
listen_address=""):
super(WebService, self).__init__(shard)
static_files = parameters.pop('static_files', [])
rpc_enabled = parameters.pop('rpc_enabled', False)
rpc_auth = parameters.pop('rpc_auth', None)
auth_middleware = parameters.pop('auth_middleware', None)
is_proxy_used = parameters.pop('is_proxy_used', None)
num_proxies_used = parameters.pop('num_proxies_used', None)
self.wsgi_app = tornado.wsgi.WSGIApplication(handlers, **parameters)
self.wsgi_app.service = self
for entry in static_files:
# TODO If we will introduce a flag to trigger autoreload in
# Jinja2 templates, use it to disable the cache arg here.
self.wsgi_app = SharedDataMiddleware(
self.wsgi_app, {"/static": entry},
cache=True, cache_timeout=SECONDS_IN_A_YEAR,
fallback_mimetype="application/octet-stream")
self.file_cacher = FileCacher(self)
self.wsgi_app = FileServerMiddleware(self.file_cacher, self.wsgi_app)
if rpc_enabled:
self.wsgi_app = DispatcherMiddleware(
self.wsgi_app, {"/rpc": RPCMiddleware(self, rpc_auth)})
# The authentication middleware needs to be applied before the
# ProxyFix as otherwise the remote address it gets is the one
# of the proxy.
if auth_middleware is not None:
self.wsgi_app = auth_middleware(self.wsgi_app)
self.auth_handler = self.wsgi_app
# If we are behind one or more proxies, we'll use the content
# of the X-Forwarded-For HTTP header (if provided) to determine
# the client IP address, ignoring the one the request came from.
# This allows to use the IP lock behind a proxy. Activate it
# only if all requests come from a trusted source (if clients
# were allowed to directlty communicate with the server they
# could fake their IP and compromise the security of IP lock).
if num_proxies_used is None:
if is_proxy_used:
num_proxies_used = 1
else:
num_proxies_used = 0
if num_proxies_used > 0:
self.wsgi_app = ProxyFix(self.wsgi_app, num_proxies_used)
self.web_server = WSGIServer((listen_address, listen_port), self)
def __call__(self, environ, start_response):
"""Execute this instance as a WSGI application.
See the PEP for the meaning of parameters. The separation of
__call__ and wsgi_app eases the insertion of middlewares.
"""
return self.wsgi_app(environ, start_response)
def run(self):
"""Start the WebService.
Both the WSGI server and the RPC server are started.
"""
self.web_server.start()
Service.run(self)
self.web_server.stop()
class OmsEventListener(object):
"""
HTTP server to get RSN OMS event notifications and do corresponding
notifications to driver/agent via callback.
"""
#
# TODO The whole asynchronous reception of external RSN OMS events needs
# to be handled in a different way, in particular, it should be a separate
# service (not a supporting class per platform driver instance) that
# listens on a well-known host:port and that publishes the corresponding
# CI events directly.
#
def __init__(self, platform_id, notify_driver_event):
"""
Creates a listener.
@param notify_driver_event callback to notify event events. Must be
provided.
"""
self._platform_id = platform_id
self._notify_driver_event = notify_driver_event
self._http_server = None
self._url = None
# _notifications: if not None, [event_instance, ...]
self._notifications = None
# _no_notifications: flag only intended for developing purposes
# see ion.agents.platform.rsn.simulator.oms_events
self._no_notifications = os.getenv("NO_OMS_NOTIFICATIONS") is not None
if self._no_notifications: # pragma: no cover
log.warn(
"%r: NO_OMS_NOTIFICATIONS env variable defined: "
"no notifications will be done", self._platform_id)
self._url = "http://NO_OMS_NOTIFICATIONS"
@property
def url(self):
"""
The URL that can be used to register a listener to the OMS.
This is None if there is no HTTP server currently running.
"""
return self._url
def keep_notifications(self, keep=True, reset=True):
"""
By default, received event notifications are not kept. Call this with
True (the default) to keep them, or with False to not keep them.
If they are currently kept and the reset param is True (the default),
then the notifications list is reinitialized.
"""
if keep:
if not self._notifications or reset:
self._notifications = []
else:
self._notifications = None
@property
def notifications(self):
"""
The current list of received notifications. This will be None if such
notifications are not being kept.
"""
return self._notifications
def start_http_server(self, host='localhost', port=0):
"""
Starts a HTTP server that handles the notification of received events.
@param host Host, by default 'localhost'.
@param port Port, by default 0 to get one dynamically.
"""
# reinitialize notifications if we are keeping them:
if self._notifications:
self._notifications = []
if self._no_notifications:
return
log.info(
"%r: starting http server for receiving event notifications at"
" %s:%s ...", self._platform_id, host, port)
try:
self._http_server = WSGIServer((host, port),
self._application,
log=sys.stdout)
self._http_server.start()
except:
log.exception(
"%r: Could not start http server for receiving event"
" notifications", self._platform_id)
raise
host_name, host_port = self._http_server.address
log.info("%r: http server started at %s:%s", self._platform_id,
host_name, host_port)
exposed_host_name = host_name
######################################################################
# adjust exposed_host_name:
# **NOTE**: the adjustment below is commented out because is not robust
# enough. For example, the use of the external name for the host would
# require the particular port to be open to the world.
# And in any case, this overall handling needs a different approach.
#
# # If the given host is 'localhost', need to get the actual hostname
# # for the exposed URL:
# if host is 'localhost':
# exposed_host_name = socket.gethostname()
######################################################################
self._url = "http://%s:%s" % (exposed_host_name, host_port)
log.info("%r: http server exposed URL = %r", self._platform_id,
self._url)
def _application(self, environ, start_response):
input = environ['wsgi.input']
body = "\n".join(input.readlines())
# log.trace('%r: notification received payload=%s', self._platform_id, body)
event_instance = yaml.load(body)
self._event_received(event_instance)
# generic OK response TODO determine appropriate variations if any
status = '200 OK'
headers = [('Content-Type', 'text/plain')]
start_response(status, headers)
return status
def _event_received(self, event_instance):
log.trace('%r: received event_instance=%s', self._platform_id,
event_instance)
if self._notifications:
self._notifications.append(event_instance)
else:
self._notifications = [event_instance]
log.debug('%r: notifying event_instance=%s', self._platform_id,
event_instance)
driver_event = ExternalEventDriverEvent(event_instance)
self._notify_driver_event(driver_event)
def stop_http_server(self):
"""
Stops the http server.
@retval the dict of received notifications or None if they are not kept.
"""
if self._http_server:
log.info("%r: HTTP SERVER: stopping http server: url=%r",
self._platform_id, self._url)
self._http_server.stop()
self._http_server = None
self._url = None
return self._notifications
class WebServer(object):
def __init__(self):
signal.signal(signal.SIGINT, self._killServer)
signal.signal(signal.SIGTERM, self._killServer)
self.wsgiserver = None
self.access_logger = None
self.restart = False
self.app = None
self.listen_address = None
self.listen_port = None
self.unix_socket_file = None
self.ssl_args = None
def init_app(self, application, config):
self.app = application
self.listen_address = config.get_config_ipaddress()
self.listen_port = config.config_port
if config.config_access_log:
log_name = "gevent.access" if _GEVENT else "tornado.access"
formatter = logger.ACCESS_FORMATTER_GEVENT if _GEVENT else logger.ACCESS_FORMATTER_TORNADO
self.access_logger, logfile = logger.create_access_log(
config.config_access_logfile, log_name, formatter)
if logfile != config.config_access_logfile:
log.warning(
"Accesslog path %s not valid, falling back to default",
config.config_access_logfile)
config.config_access_logfile = logfile
config.save()
else:
if not _GEVENT:
logger.get('tornado.access').disabled = True
certfile_path = config.get_config_certfile()
keyfile_path = config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path):
self.ssl_args = dict(certfile=certfile_path,
keyfile=keyfile_path)
else:
log.warning(
'The specified paths for the ssl certificate file and/or key file seem to be broken. '
'Ignoring ssl.')
log.warning('Cert path: %s', certfile_path)
log.warning('Key path: %s', keyfile_path)
def _make_gevent_unix_socket(self, socket_file):
# the socket file must not exist prior to bind()
if os.path.exists(socket_file):
# avoid nuking regular files and symbolic links (could be a mistype or security issue)
if os.path.isfile(socket_file) or os.path.islink(socket_file):
raise OSError(errno.EEXIST, os.strerror(errno.EEXIST),
socket_file)
os.remove(socket_file)
unix_sock = WSGIServer.get_listener(socket_file, family=socket.AF_UNIX)
self.unix_socket_file = socket_file
# ensure current user and group have r/w permissions, no permissions for other users
# this way the socket can be shared in a semi-secure manner
# between the user running calibre-web and the user running the fronting webserver
os.chmod(socket_file, 0o660)
return unix_sock
def _make_gevent_socket(self):
if os.name != 'nt':
unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET")
if unix_socket_file:
return self._make_gevent_unix_socket(
unix_socket_file), "unix:" + unix_socket_file
if self.listen_address:
return (self.listen_address, self.listen_port), None
if os.name == 'nt':
self.listen_address = '0.0.0.0'
return (self.listen_address, self.listen_port), None
try:
address = ('::', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET6)
except socket.error as ex:
log.error('%s', ex)
log.warning('Unable to listen on "", trying on IPv4 only...')
address = ('', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET)
return sock, _readable_listen_address(*address)
def _start_gevent(self):
ssl_args = self.ssl_args or {}
try:
sock, output = self._make_gevent_socket()
if output is None:
output = _readable_listen_address(self.listen_address,
self.listen_port)
log.info('Starting Gevent server on %s', output)
self.wsgiserver = WSGIServer(sock,
self.app,
log=self.access_logger,
spawn=Pool(),
**ssl_args)
if ssl_args:
wrap_socket = self.wsgiserver.wrap_socket
def my_wrap_socket(*args, **kwargs):
try:
return wrap_socket(*args, **kwargs)
except (ssl.SSLError) as ex:
log.warning('Gevent SSL Error: %s', ex)
raise GreenletExit
self.wsgiserver.wrap_socket = my_wrap_socket
self.wsgiserver.serve_forever()
finally:
if self.unix_socket_file:
os.remove(self.unix_socket_file)
self.unix_socket_file = None
def _start_tornado(self):
if os.name == 'nt' and sys.version_info > (3, 7):
import asyncio
asyncio.set_event_loop_policy(
asyncio.WindowsSelectorEventLoopPolicy())
log.info(
'Starting Tornado server on %s',
_readable_listen_address(self.listen_address, self.listen_port))
# Max Buffersize set to 200MB )
http_server = HTTPServer(WSGIContainer(self.app),
max_buffer_size=209700000,
ssl_options=self.ssl_args)
http_server.listen(self.listen_port, self.listen_address)
self.wsgiserver = IOLoop.current()
self.wsgiserver.start()
# wait for stop signal
self.wsgiserver.close(True)
def start(self):
try:
if _GEVENT:
# leave subprocess out to allow forking for fetchers and processors
self._start_gevent()
else:
self._start_tornado()
except Exception as ex:
log.error("Error starting server: %s", ex)
print("Error starting server: %s" % ex)
self.stop()
return False
finally:
self.wsgiserver = None
if not self.restart:
log.info("Performing shutdown of Calibre-Web")
# prevent irritiating log of pending tasks message from asyncio
logger.get('asyncio').setLevel(logger.logging.CRITICAL)
return True
log.info("Performing restart of Calibre-Web")
arguments = list(sys.argv)
arguments.insert(0, sys.executable)
if os.name == 'nt':
arguments = ["\"%s\"" % a for a in arguments]
os.execv(sys.executable, arguments)
return True
def _killServer(self, __, ___):
self.stop()
def stop(self, restart=False):
from . import updater_thread
updater_thread.stop()
log.info("webserver stop (restart=%s)", restart)
self.restart = restart
if self.wsgiserver:
if _GEVENT:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback_from_signal(self.wsgiserver.stop)
from gevent import monkey
from gevent.pywsgi import WSGIServer
monkey.patch_all()
from flask import Flask
import logging
from logging.handlers import RotatingFileHandler
from multiprocessing import cpu_count, Process,Manager
app = Flask(__name__)
server = WSGIServer(('0.0.0.0', 6000), app, log=None)
server.start()
handler = RotatingFileHandler('app.log',encoding='UTF-8',maxBytes=1024*1024*10,backupCount=20)
# handler = logging.FileHandler('app.log', encoding='UTF-8')
logging_format = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(lineno)s - %(message)s')
handler.setFormatter(logging_format)
app.logger.addHandler(handler)
@app.route('/',methods=['POST','GET'])
def hello():
return 'hello world'
def serve_forever():
global server
server.start_accepting()
server._stop_event.wait()
if __name__ == "__main__":
count = cpu_count()
print("cpu count %s" % count)
for i in range(count):
class OmsEventListener(object):
"""
HTTP server to get RSN OMS event notifications and do corresponding
notifications to driver/agent via callback.
"""
def __init__(self, notify_driver_event):
"""
Creates a listener.
@param notify_driver_event callback to notify event events. Must be
provided.
"""
assert notify_driver_event, "notify_driver_event callback must be provided"
self._notify_driver_event = notify_driver_event
self._http_server = None
self._url = None
# _notifications: if not None, { event_type: [event_instance, ...], ...}
self._notifications = None
@property
def url(self):
"""
The URL that can be used to register a listener to the OMS.
This is None if there is no HTTP server currently running.
"""
return self._url
def keep_notifications(self, keep=True, reset=True):
"""
By default, received event notifications are not kept. Call this with
True (the default) to keep them, or with False to not keep them.
If they are currently kept and the reset param is True (the default),
then the notifications dict is reinitialized.
"""
if keep:
if not self._notifications or reset:
self._notifications = {}
else:
self._notifications = None
@property
def notifications(self):
"""
The current dict of received notifications. This will be None if such
notifications are not being kept.
"""
return self._notifications
def start_http_server(self, host='localhost', port=0):
"""
Starts a HTTP server that handles the notification of received events.
@param host by default 'localhost'
@param port by default 0 to get one dynamically.
"""
# reinitialize notifications if we are keeping them:
if self._notifications:
self._notifications.clear()
self._http_server = WSGIServer((host, port), self.__application)
log.info("starting http server for receiving event notifications...")
self._http_server.start()
self._url = "http://%s:%s" % self._http_server.address
log.info("http server started: url=%r", self._url)
def __application(self, environ, start_response):
input = environ['wsgi.input']
body = "\n".join(input.readlines())
# log.trace('notification received payload=%s', body)
event_instance = yaml.load(body)
log.trace('notification received event_instance=%s', event_instance)
if not 'url' in event_instance:
log.warn("expecting 'url' entry in notification call")
return
if not 'ref_id' in event_instance:
log.warn("expecting 'ref_id' entry in notification call")
return
url = event_instance['url']
event_type = event_instance['ref_id']
if self._url == url:
self._event_received(event_type, event_instance)
else:
log.warn(
"got notification call with an unexpected url=%s (expected url=%s)",
url, self._url)
# generic OK response TODO determine appropriate variations if any
status = '200 OK'
headers = [('Content-Type', 'text/plain')]
start_response(status, headers)
return event_type
def _event_received(self, event_type, event_instance):
log.trace('received event_instance=%s', event_instance)
if self._notifications:
if event_type in self._notifications:
self._notifications[event_type].append(event_instance)
else:
self._notifications[event_type] = [event_instance]
log.debug('notifying event_instance=%s', event_instance)
driver_event = ExternalEventDriverEvent(event_type, event_instance)
self._notify_driver_event(driver_event)
def stop_http_server(self):
"""
Stops the http server.
@retval the dict of received notifications or None if they are not kept.
"""
if self._http_server:
log.info("HTTP SERVER: stopping http server: url=%r", self._url)
self._http_server.stop()
self._http_server = None
self._url = None
return self._notifications
#!/usr/bin/env python
# coding: utf-8
from gevent.pywsgi import WSGIServer
from geventwebsocket.handler import WebSocketHandler
from gevent.event import Event
from app import my_app_wrapper
stopper = Event()
# as suggested by ThiefMaster in:
# http://stackoverflow.com/questions/15932298/how-to-stop-a-flask-server-running-gevent-socketio
#
if __name__ == "__main__":
http_server = WSGIServer(("", 8000), my_app_wrapper(stopper), handler_class=WebSocketHandler)
http_server.start()
try:
stopper.wait()
except KeyboardInterrupt:
print ""
def run(self):
# Array since we need to pass by ref
latest_task_complete = [time()]
latest_maintenance_complete = time()
active_upload_counter = ThreadsafeCounter()
task_list = TaskList()
logging.info("Starting server on %d..." % self._port)
logging.debug("task completed callback %s", latest_task_complete)
def completed_callback(latest_task_complete):
logging.debug("task completed callback %s", latest_task_complete)
latest_task_complete[:] = [time()]
workers = Workers(self._repo,
partial(completed_callback, latest_task_complete))
workers.start(task_list, self._workers)
def webapp_callback(task_name, filepath):
task_list.add_task(Task(task_name, filepath))
webapp = UploadWebApp(__name__, active_upload_counter, webapp_callback)
http_server = WSGIServer(('', self._port), webapp)
http_server.start()
logging.info("Server started on %s" % self._port)
# loop until interrupted
while True:
try:
gsleep(5)
task_list.join()
logging.debug("task queue empty, " +
str(active_upload_counter.get_count()) +
" uploads ongoing")
time_since_maintenance = time() - latest_maintenance_complete
time_since_task = time() - latest_task_complete[0]
logging.debug("{:.1f} complete".format(time_since_task))
logging.debug(
"{:.1f} since last task, {:.1f} since last maintenance".
format(time_since_task, time_since_maintenance))
if time_since_maintenance > time_since_task:
# uploads have been processed since last maintenance
logging.debug("maintenance needed")
if time_since_task >= MAINTENANCE_WAIT:
logging.debug("idle, do maintenance")
workers.stop()
try:
output = check_output([
"flatpak", "build-update-repo",
"--generate-static-deltas", "--prune",
self._repo
],
stderr=STDOUT)
logging.info("completed maintenance: " + output)
except CalledProcessError as e:
logging.error("failed maintenance: " + e.output)
latest_maintenance_complete = time()
workers.start(task_list, self._workers)
except (KeyboardInterrupt, SystemExit):
break
logging.info("Cleaning up resources...")
http_server.stop()
workers.stop()
cthulhu = Manager()
cthulhu_started = False
while not cthulhu_started:
try:
if not cthulhu_started:
cthulhu_started = cthulhu.start()
except Exception, e:
log.exception('It borked')
log.error(str(e))
complete.wait(timeout=5)
app = get_internal_wsgi_application()
wsgi = WSGIServer(('0.0.0.0', 8002), app, **ssl)
wsgi.start()
def shutdown():
complete.set()
gevent.signal(signal.SIGTERM, shutdown)
gevent.signal(signal.SIGINT, shutdown)
x = 0
while not complete.is_set():
cthulhu.eventer.on_tick()
complete.wait(timeout=TIMEOUT)
if x > SALT_RESET_PERIOD:
cthulhu.eventer.reset_event_sink()
x = 0
x += TIMEOUT
class AccessControlTestCase(unittest.TestCase):
def setUp(self):
caliope_server.app.config['TESTING'] = True
caliope_server.init_flask_app()
caliope_server.configure_server_and_app("../../conf/test_caliope_server.json")
caliope_server.configure_logger("../../conf/tests_logger.json")
caliope_server.register_modules()
caliope_server.app.storekv = RedisStore(redis.StrictRedis())
self.http_server = WSGIServer((caliope_server.app.config['address'],
caliope_server.app.config['port']),
caliope_server.app,
handler_class=WebSocketHandler) # @IgnorePep8
self.http_server.start()
DefaultDatabase().test_defaultUserGroupOne()
def tearDown(self):
"""Get rid of the database again after each test."""
if self.rpc_client:
self.rpc_client.transport.close()
self.http_server.stop()
self.http_server = None
caliope_server.app = Flask('caliope_server')
#:Delete database
neo4j.GraphDatabaseService().clear()
def login(self, username='******', password='******'):
self.rpc_client = RPCClient(JSONRPCProtocol(),
HttpWebSocketClientTransport('ws://localhost:9001/api/ws'))
self.loginManager = self.rpc_client.get_proxy("login.")
hashed_password = hashlib.sha256(password).hexdigest()
return self.loginManager.authenticate(username=username,
password=hashed_password)
def logout(self, uuid):
if self.loginManager is None:
return
return self.loginManager.logout(uuid=uuid)
def test_login(self):
rv = self.login(u'user', u'123')
assert 'login' in rv
assert rv['login'] is True
assert 'user_uuid' in rv
assert 'session_uuid' in rv
def test_logout(self):
uuid = self.login(u'user', u'123')['user_uuid']['value']
rv = self.logout(uuid=uuid)
assert 'logout' in rv
assert rv['logout']
def test_isAccessGranted(self):
# TODO(nel): Deprecate this method.
self.login()
ac_proxy = self.rpc_client.get_proxy("ac.")
self.assertEqual({'granted': True}, ac_proxy.isAccessGranted({}))
def test_getUserList(self):
self.login()
ac_proxy = self.rpc_client.get_proxy("ac.")
user_list = set(['revisor_1', 'revisor_2', 'revisor_3', 'recepcionista_1',
'recepcionista_2', 'superuser', 'secretaria_1', 'reportero_1',
'reportero_2', 'gerente_1'])
for user in ac_proxy.getUserList({}):
self.assertIn(user, user_list)
user_list.remove(user)
self.assertEqual(user_list, set())
def test_getGroupList(self):
self.login()
ac_proxy = self.rpc_client.get_proxy("ac.")
group_list = set(['everybody', 'secretarias', 'revisores', 'reportes', 'superusers', 'gerentes', 'recepcionistas'])
for group in ac_proxy.getGroupList({}):
self.assertIn(group, group_list)
group_list.remove(group)
self.assertEqual(group_list, set())
def test_getGroupsForUser(self):
self.login()
ac_proxy = self.rpc_client.get_proxy('ac.')
print ac_proxy.getGroupsOfUser('gerente_1')
def test_getUserPermissions(self):
self.login()
permissions_of_user = \
set([('read', 'form', 'everybody'), ('read', 'form', 'gerentes'),
('read', 'document', 'everybody'), ('read', 'document', 'gerentes'),
('read', 'task', 'everybody'), ('read', 'task', 'gerentes'),
('read', 'report', 'everybody'), ('read', 'report', 'gerentes'),
('write', 'form', 'everybody'), ('write', 'form', 'gerentes'),
('write', 'document', 'everybody'), ('write', 'document', 'gerentes'),
('write', 'task', 'everybody'), ('write', 'task', 'gerentes'),
('write', 'report', 'everybody'), ('write', 'report', 'gerentes'),
('assign', 'form', 'everybody'), ('assign', 'form', 'gerentes'),
('assign', 'document', 'everybody'), ('assign', 'document', 'gerentes'),
('assign', 'task', 'everybody'), ('assign', 'task', 'gerentes'),
('assign', 'report', 'everybody'), ('assign', 'report', 'gerentes'),
('assign', 'form', 'reportes'), ('assign', 'document', 'reportes'),
('assign', 'task', 'reportes'), ('assign', 'report', 'reportes')])
ac_proxy = self.rpc_client.get_proxy("ac.")
for perm in ac_proxy.getUserPermissions('gerente_1'):
self.assertIn(tuple(perm), permissions_of_user)
permissions_of_user.remove(tuple(perm))
self.assertEqual(permissions_of_user, set())
class server:
wsgiserver = None
restart= False
def __init__(self):
signal.signal(signal.SIGINT, self.killServer)
signal.signal(signal.SIGTERM, self.killServer)
def start_gevent(self):
try:
ssl_args = dict()
certfile_path = web.ub.config.get_config_certfile()
keyfile_path = web.ub.config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path):
ssl_args = {"certfile": certfile_path,
"keyfile": keyfile_path}
else:
web.app.logger.info('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s' % (certfile_path, keyfile_path))
if os.name == 'nt':
self.wsgiserver= WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
else:
self.wsgiserver = WSGIServer(('', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
web.py3_gevent_link = self.wsgiserver
self.wsgiserver.serve_forever()
except SocketError:
try:
web.app.logger.info('Unable to listen on \'\', trying on IPv4 only...')
self.wsgiserver = WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
web.py3_gevent_link = self.wsgiserver
self.wsgiserver.serve_forever()
except (OSError, SocketError) as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
except Exception:
web.app.logger.info("Unknown error while starting gevent")
def startServer(self):
if gevent_present:
web.app.logger.info('Starting Gevent server')
# leave subprocess out to allow forking for fetchers and processors
self.start_gevent()
else:
try:
ssl = None
web.app.logger.info('Starting Tornado server')
certfile_path = web.ub.config.get_config_certfile()
keyfile_path = web.ub.config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path):
ssl = {"certfile": certfile_path,
"keyfile": keyfile_path}
else:
web.app.logger.info('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s' % (certfile_path, keyfile_path))
# Max Buffersize set to 200MB
http_server = HTTPServer(WSGIContainer(web.app),
max_buffer_size = 209700000,
ssl_options=ssl)
http_server.listen(web.ub.config.config_port)
self.wsgiserver=IOLoop.instance()
self.wsgiserver.start()
# wait for stop signal
self.wsgiserver.close(True)
except SocketError as e:
web.app.logger.info("Error starting server: %s" % e.strerror)
print("Error starting server: %s" % e.strerror)
web.helper.global_WorkerThread.stop()
sys.exit(1)
# ToDo: Somehow caused by circular import under python3 refactor
if sys.version_info > (3, 0):
self.restart = web.py3_restart_Typ
if self.restart == True:
web.app.logger.info("Performing restart of Calibre-Web")
web.helper.global_WorkerThread.stop()
if os.name == 'nt':
arguments = ["\"" + sys.executable + "\""]
for e in sys.argv:
arguments.append("\"" + e + "\"")
os.execv(sys.executable, arguments)
else:
os.execl(sys.executable, sys.executable, *sys.argv)
else:
web.app.logger.info("Performing shutdown of Calibre-Web")
web.helper.global_WorkerThread.stop()
sys.exit(0)
def setRestartTyp(self,starttyp):
self.restart = starttyp
# ToDo: Somehow caused by circular import under python3 refactor
web.py3_restart_Typ = starttyp
def killServer(self, signum, frame):
self.stopServer()
def stopServer(self):
# ToDo: Somehow caused by circular import under python3 refactor
if sys.version_info > (3, 0):
if not self.wsgiserver:
if gevent_present:
self.wsgiserver = web.py3_gevent_link
else:
self.wsgiserver = IOLoop.instance()
if self.wsgiserver:
if gevent_present:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback(self.wsgiserver.stop)
@staticmethod
def getNameVersion():
if gevent_present:
return {'Gevent':'v'+geventVersion}
else:
return {'Tornado':'v'+tornadoVersion}
class CaliopeServerTestCase(unittest.TestCase):
def setUp(self):
caliope_server.app.config['TESTING'] = True
caliope_server.init_flask_app()
caliope_server.configure_server_and_app(
"../../conf/test_caliope_server.json")
caliope_server.configure_logger("../../conf/tests_logger.json")
caliope_server.register_modules()
caliope_server.app.storekv = RedisStore(redis.StrictRedis())
self.http_server = WSGIServer(
(caliope_server.app.config['address'],
caliope_server.app.config['port']),
caliope_server.app,
handler_class=WebSocketHandler) # @IgnorePep8
self.http_server.start()
self.create_default_database()
def tearDown(self):
"""Get rid of the database again after each test."""
if self.rpc_client:
self.rpc_client.transport.close()
self.http_server.stop()
self.http_server = None
caliope_server.app = Flask('caliope_server')
#:Delete database
neo4j.GraphDatabaseService().clear()
def create_default_database(self):
DefaultDatabase().test_defaultUserGroupOne()
def login(self, username, password):
self.rpc_client = RPCClient(
JSONRPCProtocol(),
HttpWebSocketClientTransport('ws://localhost:9001/api/ws'))
self.loginManager = self.rpc_client.get_proxy("login.")
hashed_password = hashlib.sha256(password).hexdigest()
return self.loginManager.authenticate(username=username,
password=hashed_password)
def logout(self, uuid):
if self.loginManager is None:
return
return self.loginManager.logout(uuid=uuid)
def test_login(self):
rv = self.login(u'user', u'123')
expected = \
{u'first_name': {u'value': u'User'},
u'last_name': {u'value': u'Test'},
u'image': {u'data': None},
u'user': {u'value': u'user'},
u'login': True,
}
self.assertDictContainsSubset(expected, rv)
self.assertIn("session_uuid", rv)
self.assertIn("user_uuid", rv)
def test_logout(self):
uuid = self.login(u'user', u'123')['user_uuid']['value']
rv = self.logout(uuid=uuid)
self.assertIn('logout', rv)
self.assertTrue(rv['logout'])
self.assertIn('uuid', rv)
self.assertEqual(uuid, rv['uuid'])
def test_accounts_get_public_info(self):
users = [self.login(u'user', u'123')['user_uuid']['value']]
accounts_proxy = self.rpc_client.get_proxy(prefix="accounts.")
info = accounts_proxy.getPublicInfo(users)
assert len(info) == 1
assert 'uuid' in info[0]
for user in users:
info_uuid = info[0]['uuid']['value']
assert user == info_uuid
def test_projects_create(self):
user = self.login(u'user', u'123')
projects_proxy = self.rpc_client.get_proxy(prefix="project.")
model = projects_proxy.getModel()
data = {
"name":
"PROYECTO 305",
"general_location":
"<p><em><strong>ASDASDASD</strong></em><br></p>",
"locality":
"suba",
"project_type":
"py_gr_escala",
"profit_center":
"ASDASDADS",
"areas": [{
"tipo": "A1",
"valor": "121"
}, {
"tipo": "A2",
"valor": "13"
}],
"uuid":
model['data']['uuid']['value']
}
#: TODO Check for real asserts
try:
rv = projects_proxy.create(data=data)
assert True
except BaseException:
assert False
def test_form_find(self):
user = self.login(u'user', u'123')
projects_proxy = self.rpc_client.get_proxy(prefix="project.")
rv = projects_proxy.getAll()
self.assertIsNotNone(rv)
class TaskServicesTestCase(unittest.TestCase):
def setUp(self):
caliope_server.app.config['TESTING'] = True
caliope_server.init_flask_app()
caliope_server.configure_server_and_app(
"conf/test_caliope_server.json")
caliope_server.configure_logger("conf/tests_logger.json")
caliope_server.register_modules()
caliope_server.app.storekv = RedisStore(redis.StrictRedis())
self.http_server = WSGIServer((caliope_server.app.config['address'],
caliope_server.app.config['port']),
caliope_server.app,
handler_class=WebSocketHandler) # @IgnorePep8
self.http_server.start()
self.nodes_created = set()
self.create_default_database()
def tearDown(self):
"""Get rid of the database again after each test."""
if self.rpc_client:
self.rpc_client.transport.close()
self.http_server.stop()
self.http_server = None
caliope_server.app = Flask('caliope_server')
#:Delete database
self.remove_used_nodes(self.nodes_created)
def create_default_database(self):
self.create_UserGroup('user', 'group')
def create_UserGroup(self, username, groupname):
try:
u1 = CaliopeUser()
u1.username = username
u1.password = hashlib.sha256(u'123').hexdigest()
u1.domainname = 'correlibre.org'
u1.first_name = "User"
u1.last_name = "Test"
u1.avatar = "common-img/avatar.png"
u1.save()
self.nodes_created.add(u1.uuid)
g1 = CaliopeGroup()
g1.name = groupname
g1.code = groupname
g1.save()
self.nodes_created.add(g1.uuid)
u1.member_of.connect(g1)
g1.members.connect(u1)
self.assertTrue(u1.member_of.is_connected(g1))
self.assertTrue(g1.members.is_connected(u1))
except UniqueProperty:
try:
u1 = CaliopeUser.index.get(username=username)
g1 = CaliopeGroup.index.get(code=groupname)
assert u1 is not None and g1 is not None
assert u1.member_of.is_connected(g1)
assert g1.members.is_connected(u1)
except DoesNotExist:
assert False
def remove_used_nodes(self, node_list):
query = "START n=node:CaliopeStorage('uuid:{}') " \
"MATCH n-[r]-() " \
"DELETE n, r"
batch_query = []
for uuid in node_list:
batch_query.append(neo4j.cypher.Query(neo4j.GraphDatabaseService(),
query.format(uuid)).execute())
def login(self, username, password):
self.rpc_client = RPCClient(JSONRPCProtocol(),
HttpWebSocketClientTransport(
'ws://localhost:9001/api/ws'))
self.loginManager = self.rpc_client.get_proxy("login.")
hashed_password = hashlib.sha256(password).hexdigest()
return self.loginManager.authenticate(username=username,
password=hashed_password)
def logout(self, uuid):
if self.loginManager is None:
return
return self.loginManager.logout(uuid=uuid)
def test_login(self):
rv = self.login(u'user', u'123')
expected = \
{u'first_name': {u'value': u'User'},
u'last_name': {u'value': u'Test'},
u'image': {u'data': None},
u'user': {u'value': u'user'},
u'login': True,
}
self.assertDictContainsSubset(expected, rv)
self.assertIn("session_uuid", rv)
self.assertIn("user_uuid", rv)
def test_logout(self):
uuid = self.login(u'user', u'123')['user_uuid']['value']
rv = self.logout(uuid=uuid)
self.assertIn('logout', rv)
self.assertTrue(rv['logout'])
self.assertIn('uuid', rv)
self.assertEqual(uuid, rv['uuid'])
def test_ts_get_model(self):
user = self.login(u'user', u'123')
tasks_proxy = self.rpc_client.get_proxy(prefix="tasks.")
model = tasks_proxy.getModel()
self.assertIsNotNone(model)
def test_task_get_model_and_data(self):
user = self.login(u'user', u'123')
tasks_proxy = self.rpc_client.get_proxy(prefix="tasks.")
model = tasks_proxy.getModel()
self.assertIsNotNone(model)
uuid = model["data"]["uuid"]["value"]
#:update
update = tasks_proxy.updateField(uuid=uuid,
field_name="name",
value="test")
#:commit
commit = tasks_proxy.commit(uuid=uuid)
self.assertTrue(commit)
model_and_data = tasks_proxy.getModelAndData(uuid=uuid)
self.assertEqual(model_and_data["data"]["name"]["value"], "test")
def test_task_update_commit_field_single_value(self):
user = self.login(u'user', u'123')
tasks_proxy = self.rpc_client.get_proxy(prefix="tasks.")
model = tasks_proxy.getModel()
uuid = model["data"]["uuid"]["value"]
#:update
update = tasks_proxy.updateField(uuid=uuid,
field_name="name",
value="test")
#:commit
commit = tasks_proxy.commit(uuid=uuid)
self.assertTrue(commit)
#:update a commited value
update = tasks_proxy.updateField(uuid=uuid,
field_name="name",
value="foo")
self.assertTrue(update)
#:commit again a previusly commited value after being updated
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#:update twice a draft and commit
update = tasks_proxy.updateField(uuid=uuid,
field_name="name",
value="more foo")
self.assertTrue(update)
update = tasks_proxy.updateField(uuid=uuid,
field_name="name",
value="not more foo")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
self.nodes_created.add(uuid)
def test_task_update_commit_field_single_value_not_exist_in_class(self):
user = self.login(u'user', u'123')
tasks_proxy = self.rpc_client.get_proxy(prefix="tasks.")
model = tasks_proxy.getModel()
uuid = model["data"]["uuid"]["value"]
update = tasks_proxy.updateField(uuid=uuid,
field_name="other",
value="test")
self.assertTrue(update)
try:
response = tasks_proxy.commit(uuid=uuid)
except RPCError as error:
self.assertIsInstance(error, RPCError)
def test_task_update_commit_field_list(self):
user = self.login(u'user', u'123')
tasks_proxy = self.rpc_client.get_proxy(prefix="tasks.")
model = tasks_proxy.getModel()
uuid = model["data"]["uuid"]["value"]
#:update invalid index first.
try:
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
value="subtask0")
self.assertTrue(update)
except RPCError as error:
self.assertIsInstance(error, RPCError)
#:update and commit with empty lists
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=-1,
value="subtask0")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#: update and commit already commited lists
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
value="new_subtask0")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#: update twice and commit
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
value="new_subtask0_up1")
self.assertTrue(update)
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
value="new_subtask_up2")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#:update a non valid subfield_id
try:
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=1,
value="not subtask1")
except RPCError as error:
self.assertIsInstance(error, RPCError)
def test_task_update_commit_field_list_list(self):
user = self.login(u'user', u'123')
tasks_proxy = self.rpc_client.get_proxy(prefix="tasks.")
model = tasks_proxy.getModel()
uuid = model["data"]["uuid"]["value"]
#:update with invalid pos
try:
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=-1,
pos=0,
value="subtask0")
except RPCError as error:
self.assertIsInstance(error, RPCError)
#:update with invalid subfield_id
try:
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos=-1,
value="subtask0")
except RPCError as error:
self.assertIsInstance(error, RPCError)
#:update and commit with empty lists
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=-1,
pos=-1,
value="subtask0")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#: update and commit already commited lists
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos=0,
value="new_subtask0")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#update twice same, append new and commit
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos=0,
value="new_subtask0_up1")
self.assertTrue(update)
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos=0,
value="new_subtask0_up2")
self.assertTrue(update)
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos=-1,
value="new_subtask1_up2")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#:update a non valid pos
try:
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos=1,
value="not subtask1")
except RPCError as error:
self.assertIsInstance(error, RPCError)
def test_task_update_commit_field_list_dict(self):
user = self.login(u'user', u'123')
tasks_proxy = self.rpc_client.get_proxy(prefix="tasks.")
model = tasks_proxy.getModel()
uuid = model["data"]["uuid"]["value"]
#:update with invalid subfield_id
try:
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos="foo",
value="subtask0")
except RPCError as error:
self.assertIsInstance(error, RPCError)
#:update and commit with empty subfield
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=-1,
pos="foo",
value="subtask0")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#: update and commit already commited lists
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos="foo",
value="new_subtask0")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#update twice same, append new and commit
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos="foo1",
value="new_subtask0_up1")
self.assertTrue(update)
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos="foo1",
value="new_subtask0_up2")
self.assertTrue(update)
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos="foo_new",
value="new_subtask0_up2")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#:update a non valid pos
try:
update = tasks_proxy.updateField(uuid=uuid,
field_name="subtasks",
subfield_id=0,
pos=1,
value="not subtask1")
except RPCError as error:
self.assertIsInstance(error, RPCError)
def test_task_update_commit_field_dict(self):
user = self.login(u'user', u'123')
tasks_proxy = self.rpc_client.get_proxy(prefix="tasks.")
model = tasks_proxy.getModel()
uuid = model["data"]["uuid"]["value"]
#:update and commint for fist time
update = tasks_proxy.updateField(uuid=uuid,
field_name="comments",
subfield_id="user1",
value="comment0")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#: update and commit already commited lists
update = tasks_proxy.updateField(uuid=uuid,
field_name="comments",
subfield_id="user1",
value="comment0_up0")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#: update twice, append new and commit
update = tasks_proxy.updateField(uuid=uuid,
field_name="comments",
subfield_id="user1",
value="comment0_up1")
self.assertTrue(update)
update = tasks_proxy.updateField(uuid=uuid,
field_name="comments",
subfield_id="user1",
value="comment0_up2")
self.assertTrue(update)
update = tasks_proxy.updateField(uuid=uuid,
field_name="comments",
subfield_id="user2",
value="comment0")
self.assertTrue(update)
self.assertTrue(tasks_proxy.commit(uuid=uuid))
#:update a non valid subfield_id
try:
update = tasks_proxy.updateField(uuid=uuid,
field_name="comments",
subfield_id=1,
value="not subtask1")
except RPCError as error:
self.assertIsInstance(error, RPCError)
class APIServer():
_api_prefix = '/api/1'
def __init__(self,
rest_api: RestAPI,
cors_domain_list: List[str] = None) -> None:
flask_app = Flask(__name__)
if cors_domain_list:
CORS(flask_app, origins=cors_domain_list)
blueprint = create_blueprint()
flask_api_context = Api(blueprint, prefix=self._api_prefix)
setup_urls(
flask_api_context=flask_api_context,
rest_api=rest_api,
urls=URLS_V1,
)
self.rest_api = rest_api
self.flask_app = flask_app
self.blueprint = blueprint
self.wsgiserver: Optional[WSGIServer] = None
self.flask_app.register_blueprint(self.blueprint)
self.flask_app.errorhandler(HTTPStatus.NOT_FOUND)(endpoint_not_found)
self.flask_app.register_error_handler(Exception,
self.unhandled_exception)
@staticmethod
def unhandled_exception(exception: Exception) -> Response:
""" Flask.errorhandler when an exception wasn't correctly handled """
log.critical(
"Unhandled exception when processing endpoint request",
exc_info=True,
)
return api_response(wrap_in_fail_result(str(exception)),
HTTPStatus.INTERNAL_SERVER_ERROR)
def run(self,
host: str = '127.0.0.1',
port: int = 5042,
**kwargs: Any) -> None:
"""This is only used for the data faker and not used in production"""
self.flask_app.run(host=host, port=port, **kwargs)
def start(self, host: str = '127.0.0.1', port: int = 5042) -> None:
"""This is used to start the API server in production"""
wsgi_logger = logging.getLogger(__name__ + '.pywsgi')
self.wsgiserver = WSGIServer(
(host, port),
self.flask_app,
log=wsgi_logger,
error_log=wsgi_logger,
)
msg = f'Rotki API server is running at: {host}:{port}'
print(msg)
log.info(msg)
self.wsgiserver.start()
def stop(self, timeout: int = 5) -> None:
"""Stops the API server. If handlers are running after timeout they are killed"""
if self.wsgiserver is not None:
self.wsgiserver.stop(timeout)
self.wsgiserver = None
self.rest_api.stop()
cov_port = cov_config.get('sub_port', None)
cov = coverage.Coverage()
app_cov = Flask(cov_config.get('exec_file', 'hehe'))
@app_cov.route("/", methods=['GET', 'POST'])
def index():
pass
return 'eee'
@app_cov.route("/save", methods=['GET', 'POST'])
def save():
cov.save()
return 'save ok'
@app_cov.route("/stop", methods=['GET', 'POST'])
def stop():
cov.save()
return 'stop ok'
config = {
'host': '127.0.0.1',
'port': cov_port,
# 'debug' : True,
}
http_server = WSGIServer(('127.0.0.1', cov_port), app_cov)
http_server.start()
# app.run(**config)
print('this time is ', cov_port)
cov.start()
class WebServer(object):
def __init__(self):
signal.signal(signal.SIGINT, self._killServer)
signal.signal(signal.SIGTERM, self._killServer)
self.wsgiserver = None
self.access_logger = None
self.restart = False
self.app = None
self.listen_address = None
self.listen_port = None
self.unix_socket_file = None
self.ssl_args = None
def init_app(self, application, config):
self.app = application
self.listen_address = config.get_config_ipaddress()
self.listen_port = config.config_port
if config.config_access_log:
log_name = "gevent.access" if _GEVENT else "tornado.access"
formatter = logger.ACCESS_FORMATTER_GEVENT if _GEVENT else logger.ACCESS_FORMATTER_TORNADO
self.access_logger, logfile = logger.create_access_log(
config.config_access_logfile, log_name, formatter)
if logfile != config.config_access_logfile:
log.warning(
"Accesslog path %s not valid, falling back to default",
config.config_access_logfile)
config.config_access_logfile = logfile
config.save()
else:
if not _GEVENT:
logger.get('tornado.access').disabled = True
certfile_path = config.get_config_certfile()
keyfile_path = config.get_config_keyfile()
if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path):
self.ssl_args = dict(certfile=certfile_path,
keyfile=keyfile_path)
else:
log.warning(
'The specified paths for the ssl certificate file and/or key file seem to be broken. '
'Ignoring ssl.')
log.warning('Cert path: %s', certfile_path)
log.warning('Key path: %s', keyfile_path)
def _make_gevent_unix_socket(self, socket_file):
# the socket file must not exist prior to bind()
if os.path.exists(socket_file):
# avoid nuking regular files and symbolic links (could be a mistype or security issue)
if os.path.isfile(socket_file) or os.path.islink(socket_file):
raise OSError(errno.EEXIST, os.strerror(errno.EEXIST),
socket_file)
os.remove(socket_file)
unix_sock = WSGIServer.get_listener(socket_file, family=socket.AF_UNIX)
self.unix_socket_file = socket_file
# ensure current user and group have r/w permissions, no permissions for other users
# this way the socket can be shared in a semi-secure manner
# between the user running calibre-web and the user running the fronting webserver
os.chmod(socket_file, 0o660)
return unix_sock
def _make_gevent_socket(self):
if os.name != 'nt':
unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET")
if unix_socket_file:
return self._make_gevent_unix_socket(
unix_socket_file), "unix:" + unix_socket_file
if self.listen_address:
return (self.listen_address, self.listen_port), None
if os.name == 'nt':
self.listen_address = '0.0.0.0'
return (self.listen_address, self.listen_port), None
try:
address = ('::', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET6)
except socket.error as ex:
log.error('%s', ex)
log.warning('Unable to listen on "", trying on IPv4 only...')
address = ('', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET)
return sock, _readable_listen_address(*address)
@staticmethod
def _get_args_for_reloading():
"""Determine how the script was executed, and return the args needed
to execute it again in a new process.
Code from https://github.com/pyload/pyload. Author GammaC0de, voulter
"""
rv = [sys.executable]
py_script = sys.argv[0]
args = sys.argv[1:]
# Need to look at main module to determine how it was executed.
__main__ = sys.modules["__main__"]
# The value of __package__ indicates how Python was called. It may
# not exist if a setuptools script is installed as an egg. It may be
# set incorrectly for entry points created with pip on Windows.
if getattr(__main__, "__package__", None) is None or (
os.name == "nt" and __main__.__package__ == ""
and not os.path.exists(py_script)
and os.path.exists("{}.exe".format(py_script))):
# Executed a file, like "python app.py".
py_script = os.path.abspath(py_script)
if os.name == "nt":
# Windows entry points have ".exe" extension and should be
# called directly.
if not os.path.exists(py_script) and os.path.exists(
"{}.exe".format(py_script)):
py_script += ".exe"
if (os.path.splitext(sys.executable)[1] == ".exe"
and os.path.splitext(py_script)[1] == ".exe"):
rv.pop(0)
rv.append(py_script)
else:
# Executed a module, like "python -m module".
if sys.argv[0] == "-m":
args = sys.argv
else:
if os.path.isfile(py_script):
# Rewritten by Python from "-m script" to "/path/to/script.py".
py_module = __main__.__package__
name = os.path.splitext(os.path.basename(py_script))[0]
if name != "__main__":
py_module += ".{}".format(name)
else:
# Incorrectly rewritten by pydevd debugger from "-m script" to "script".
py_module = py_script
rv.extend(("-m", py_module.lstrip(".")))
rv.extend(args)
return rv
def _start_gevent(self):
ssl_args = self.ssl_args or {}
try:
sock, output = self._make_gevent_socket()
if output is None:
output = _readable_listen_address(self.listen_address,
self.listen_port)
log.info('Starting Gevent server on %s', output)
self.wsgiserver = WSGIServer(sock,
self.app,
log=self.access_logger,
spawn=Pool(),
**ssl_args)
if ssl_args:
wrap_socket = self.wsgiserver.wrap_socket
def my_wrap_socket(*args, **kwargs):
try:
return wrap_socket(*args, **kwargs)
except (ssl.SSLError, OSError) as ex:
log.warning('Gevent SSL Error: %s', ex)
raise GreenletExit
self.wsgiserver.wrap_socket = my_wrap_socket
self.wsgiserver.serve_forever()
finally:
if self.unix_socket_file:
os.remove(self.unix_socket_file)
self.unix_socket_file = None
def _start_tornado(self):
if os.name == 'nt' and sys.version_info > (3, 7):
import asyncio
asyncio.set_event_loop_policy(
asyncio.WindowsSelectorEventLoopPolicy())
log.info(
'Starting Tornado server on %s',
_readable_listen_address(self.listen_address, self.listen_port))
# Max Buffersize set to 200MB )
http_server = HTTPServer(WSGIContainer(self.app),
max_buffer_size=209700000,
ssl_options=self.ssl_args)
http_server.listen(self.listen_port, self.listen_address)
self.wsgiserver = IOLoop.current()
self.wsgiserver.start()
# wait for stop signal
self.wsgiserver.close(True)
def start(self):
try:
if _GEVENT:
# leave subprocess out to allow forking for fetchers and processors
self._start_gevent()
else:
self._start_tornado()
except Exception as ex:
log.error("Error starting server: %s", ex)
print("Error starting server: %s" % ex)
self.stop()
return False
finally:
self.wsgiserver = None
# prevent irritating log of pending tasks message from asyncio
logger.get('asyncio').setLevel(logger.logging.CRITICAL)
if not self.restart:
log.info("Performing shutdown of Calibre-Web")
return True
log.info("Performing restart of Calibre-Web")
args = self._get_args_for_reloading()
subprocess.call(args, close_fds=True) # nosec
return True
def _killServer(self, __, ___):
self.stop()
def stop(self, restart=False):
from . import updater_thread
updater_thread.stop()
log.info("webserver stop (restart=%s)", restart)
self.restart = restart
if self.wsgiserver:
if _GEVENT:
self.wsgiserver.close()
else:
self.wsgiserver.add_callback_from_signal(self.wsgiserver.stop)
class WebService(Service):
"""RPC service with Web server capabilities.
"""
def __init__(self, listen_port, handlers, parameters, shard=0,
listen_address=""):
super(WebService, self).__init__(shard)
static_files = parameters.pop('static_files', [])
rpc_enabled = parameters.pop('rpc_enabled', False)
rpc_auth = parameters.pop('rpc_auth', None)
auth_middleware = parameters.pop('auth_middleware', None)
is_proxy_used = parameters.pop('is_proxy_used', None)
num_proxies_used = parameters.pop('num_proxies_used', None)
self.wsgi_app = tornado.wsgi.WSGIApplication(handlers, **parameters)
self.wsgi_app.service = self
for entry in static_files:
self.wsgi_app = SharedDataMiddleware(
self.wsgi_app, {"/static": entry})
if rpc_enabled:
self.wsgi_app = DispatcherMiddleware(
self.wsgi_app, {"/rpc": RPCMiddleware(self, rpc_auth)})
# The authentication middleware needs to be applied before the
# ProxyFix as otherwise the remote address it gets is the one
# of the proxy.
if auth_middleware is not None:
self.wsgi_app = auth_middleware(self.wsgi_app)
self.auth_handler = self.wsgi_app
# If we are behind one or more proxies, we'll use the content
# of the X-Forwarded-For HTTP header (if provided) to determine
# the client IP address, ignoring the one the request came from.
# This allows to use the IP lock behind a proxy. Activate it
# only if all requests come from a trusted source (if clients
# were allowed to directlty communicate with the server they
# could fake their IP and compromise the security of IP lock).
if num_proxies_used is None:
if is_proxy_used:
num_proxies_used = 1
else:
num_proxies_used = 0
if num_proxies_used > 0:
self.wsgi_app = ProxyFix(self.wsgi_app, num_proxies_used)
self.web_server = WSGIServer((listen_address, listen_port),
self.wsgi_app)
def run(self):
"""Start the WebService.
Both the WSGI server and the RPC server are started.
"""
self.web_server.start()
Service.run(self)
self.web_server.stop()
class WebWriter(object):
def __init__(self, halt_event):
self._log = logging.getLogger("WebWriter")
memcached_client = memcache.Client(_memcached_nodes)
self._interaction_pool = gdbpool.interaction_pool.DBInteractionPool(
get_central_database_dsn(),
pool_name=_central_pool_name,
pool_size=_database_pool_size,
do_log=True)
authenticator = InteractionPoolAuthenticator(memcached_client,
self._interaction_pool)
# Ticket #25: must run database operation in a greenlet
greenlet = gevent.Greenlet.spawn(_get_cluster_row_and_node_row,
self._interaction_pool)
greenlet.join()
self._cluster_row, node_row = greenlet.get()
self._unified_id_factory = UnifiedIDFactory(node_row.id)
self._deliverator = Deliverator()
self._zeromq_context = zmq.Context()
self._pull_server = GreenletPULLServer(self._zeromq_context,
_web_writer_pipeliner_address,
self._deliverator)
self._pull_server.link_exception(self._unhandled_greenlet_exception)
self._data_writer_clients = list()
for node_name, address in zip(_node_names, _data_writer_addresses):
resilient_client = GreenletResilientClient(
self._zeromq_context,
node_name,
address,
_client_tag,
_web_writer_pipeliner_address,
self._deliverator,
connect_messages=[])
resilient_client.link_exception(self._unhandled_greenlet_exception)
self._data_writer_clients.append(resilient_client)
self._space_accounting_dealer_client = GreenletDealerClient(
self._zeromq_context, _local_node_name,
_space_accounting_server_address)
self._space_accounting_dealer_client.link_exception(
self._unhandled_greenlet_exception)
push_client = GreenletPUSHClient(
self._zeromq_context,
_local_node_name,
_space_accounting_pipeline_address,
)
self._accounting_client = SpaceAccountingClient(
_local_node_name, self._space_accounting_dealer_client,
push_client)
self._event_push_client = EventPushClient(self._zeromq_context,
"web-server")
# message sent to data writers telling them the server
# is (re)starting, thereby invalidating any archives
# that are in progress for this node
unified_id = self._unified_id_factory.next()
timestamp = create_timestamp()
self._event_push_client.info("web-writer-start",
"web writer (re)start",
unified_id=unified_id,
timestamp_repr=repr(timestamp),
source_node_name=_local_node_name)
id_translator_keys_path = os.environ.get(
"NIMBUS_IO_ID_TRANSLATION_KEYS",
os.path.join(_repository_path, "id_translator_keys.pkl"))
with open(id_translator_keys_path, "r") as input_file:
id_translator_keys = pickle.load(input_file)
self._id_translator = InternalIDTranslator(
id_translator_keys["key"], id_translator_keys["hmac_key"],
id_translator_keys["iv_key"], id_translator_keys["hmac_size"])
redis_queue = gevent.queue.Queue()
self._redis_sink = OperationalStatsRedisSink(halt_event, redis_queue,
_local_node_name)
self._redis_sink.link_exception(self._unhandled_greenlet_exception)
self.application = Application(self._cluster_row,
self._unified_id_factory,
self._id_translator,
self._data_writer_clients,
authenticator, self._accounting_client,
self._event_push_client, redis_queue)
self.wsgi_server = WSGIServer((_web_writer_host, _web_writer_port),
application=self.application,
backlog=_wsgi_backlog)
def start(self):
self._space_accounting_dealer_client.start()
self._pull_server.start()
for client in self._data_writer_clients:
client.start()
self._redis_sink.start()
self.wsgi_server.start()
def stop(self):
self._log.info("stopping wsgi web server")
self.wsgi_server.stop()
self._accounting_client.close()
self._log.debug("killing greenlets")
self._space_accounting_dealer_client.kill()
self._pull_server.kill()
for client in self._data_writer_clients:
client.kill()
self._redis_sink.kill()
self._log.debug("joining greenlets")
self._space_accounting_dealer_client.join()
self._pull_server.join()
for client in self._data_writer_clients:
client.join()
self._redis_sink.kill()
self._log.debug("closing zmq")
self._event_push_client.close()
self._zeromq_context.term()
def _unhandled_greenlet_exception(self, greenlet_object):
try:
greenlet_object.get()
except Exception:
self._log.exception(str(greenlet_object))
exctype, value = sys.exc_info()[:2]
self._event_push_client.exception("unhandled_greenlet_exception",
str(value),
exctype=exctype.__name__)
class APIServer:
"""
Runs the API-server that routes the endpoint to the resources.
The API is wrapped in multiple layers, and the Server should be invoked this way::
# instance of the raiden-api
raiden_api = RaidenAPI(...)
# wrap the raiden-api with rest-logic and encoding
rest_api = RestAPI(raiden_api)
# create the server and link the api-endpoints with flask / flask-restful middleware
api_server = APIServer(rest_api)
# run the server
api_server.run('127.0.0.1', 5001, debug=True)
"""
_api_prefix = '/api/1'
def __init__(self, rest_api, cors_domain_list=None, web_ui=False, eth_rpc_endpoint=None):
if rest_api.version != 1:
raise ValueError(
'Invalid api version: {}'.format(rest_api.version)
)
flask_app = Flask(__name__)
if cors_domain_list:
CORS(flask_app, origins=cors_domain_list)
if eth_rpc_endpoint:
if not eth_rpc_endpoint.startswith('http'):
eth_rpc_endpoint = 'http://{}'.format(eth_rpc_endpoint)
flask_app.config['WEB3_ENDPOINT'] = eth_rpc_endpoint
blueprint = create_blueprint()
flask_api_context = Api(blueprint, prefix=self._api_prefix)
restapi_setup_type_converters(
flask_app,
{'hexaddress': HexAddressConverter},
)
restapi_setup_urls(
flask_api_context,
rest_api,
URLS_V1,
)
self.rest_api = rest_api
self.flask_app = flask_app
self.blueprint = blueprint
self.flask_api_context = flask_api_context
self.wsgiserver = None
self.flask_app.register_blueprint(self.blueprint)
self.flask_app.config['WEBUI_PATH'] = '../ui/web/dist/'
if is_frozen():
# Inside frozen pyinstaller image
self.flask_app.config['WEBUI_PATH'] = '{}/raiden/ui/web/dist/'.format(sys.prefix)
self.flask_app.errorhandler(HTTPStatus.NOT_FOUND)(endpoint_not_found)
if web_ui:
for route in ('/ui/<path:file_name>', '/ui', '/ui/', '/index.html', '/'):
self.flask_app.add_url_rule(
route,
route,
view_func=self._serve_webui,
methods=('GET', ),
)
def _serve_webui(self, file_name='index.html'): # pylint: disable=redefined-builtin
try:
assert file_name
web3 = self.flask_app.config.get('WEB3_ENDPOINT')
if web3 and 'config.' in file_name and file_name.endswith('.json'):
host = request.headers.get('Host')
if any(h in web3 for h in ('localhost', '127.0.0.1')) and host:
_, _port = split_endpoint(web3)
_host, _ = split_endpoint(host)
web3 = 'http://{}:{}'.format(_host, _port)
response = jsonify({'raiden': self._api_prefix, 'web3': web3})
else:
response = send_from_directory(self.flask_app.config['WEBUI_PATH'], file_name)
except (NotFound, AssertionError):
response = send_from_directory(self.flask_app.config['WEBUI_PATH'], 'index.html')
return response
def run(self, host='127.0.0.1', port=5001, **kwargs):
self.flask_app.run(host=host, port=port, **kwargs)
def start(self, host='127.0.0.1', port=5001):
# WSGI expects a stdlib logger, with structlog there's conflict of method names
wsgi_log = logging.getLogger(__name__ + '.pywsgi')
self.wsgiserver = WSGIServer(
(host, port),
self.flask_app,
log=wsgi_log,
error_log=wsgi_log
)
self.wsgiserver.start()
def stop(self, timeout=5):
if getattr(self, 'wsgiserver', None):
self.wsgiserver.stop(timeout)
self.wsgiserver = None
class Robot:
"""
A robot is the main context where the templates and service lives.
It is responsible to:
- run the REST API server
- download the template from a git repository
- load the templates in memory and make them available
"""
def __init__(self):
self._stop_event = Event()
self._stop_event.set()
self.data_repo_url = None
self._http = None # server handler
self._addr = None
self._sig_handler = []
@property
def started(self):
return not self._stop_event.is_set()
@property
def address(self):
return self._addr
def set_data_repo(self, url):
"""
Set the data repository used to serialize services state.
@param path: can be a git URL or a absolute path or None
if git url: clone the repository locally, and use it as configuration repo
if absolute path: make sure the directory exist and use it as configuration repo
if None: automatically create a configuration repository in `{j.dirs.DATADIR}/zrobot`
It can be the same of one of the template repository used.
"""
config.data_repo = config.DataRepo(url)
self.data_repo_url = url
def add_template_repo(self, url, directory='templates'):
url, branch = giturl.parse_template_repo_url(url)
tcol.add_repo(url=url, branch=branch, directory=directory)
def set_config_repo(self, url=None, key=None):
"""
make sure the jumpscale configuration repository is initialized
@param path: can be a git URL or a absolute path or None
if git url: clone the repository locally, and use it as configuration repo
if absolute path: make sure the directory exist and use it as configuration repo
if None: automatically create a configuration repository in `{j.dirs.CODEDIR}/local/stdorg/config`
@param key: path to the sshkey to use with the configuration repo
if key is None, a key is automatically generated
"""
config.config_repo = config.ConfigRepo(url=url, key=key)
def start(self,
listen=":6600",
log_level=logging.DEBUG,
block=True,
auto_push=False,
auto_push_interval=60,
admin_organization=None,
user_organization=None,
mode=None,
god=False,
**kwargs):
"""
start the rest web server
load the services from the local git repository
"""
self._stop_event.clear()
config.mode = mode
config.god = god # when true, this allow to get data and logs from services using the REST API
if config.data_repo is None:
raise RuntimeError(
"Not data repository set. Robot doesn't know where to save data."
)
if not j.tools.configmanager.path:
raise RuntimeError(
"config manager is not configured, can't continue")
# configure storage
storage.init(config)
# instantiate webhooks manager and load the configured webhooks
config.webhooks = webhooks.get(config)
logger = j.logger.get('zerorobot')
logger.info("data directory: %s" % config.data_repo.path)
logger.info("config directory: %s" % j.tools.configmanager.path)
keypath = j.tools.configmanager.keypath
if not keypath:
keypath = os.path.expanduser(
os.path.join('~/.ssh', j.tools.configmanager.keyname))
logger.info("sshkey used: %s" % keypath)
# configure prometheus monitoring
if not kwargs.get('testing', False):
monitor(app)
# configure authentication middleware
_configure_authentication(admin_organization, user_organization)
for sig in [signal.SIGINT, signal.SIGTERM]:
self._sig_handler.append(gevent.signal(sig, self.stop))
# configure logger
app._logger = logger
# auto-push data repo
if auto_push:
logger.info("auto push of data repo enabled")
config.data_repo.start_auto_push(interval=auto_push_interval,
logger=logger)
config.config_repo.start_auto_push(interval=auto_push_interval,
logger=logger)
# load services from data repo
loader.load_services(config)
# notify services that they can start processing their task list
config.SERVICE_LOADED.set()
if mode == 'node':
_create_node_service()
# only keep executed tasks for 2 hours
gevent.spawn(_trim_tasks, 7200)
# using a pool allow to kill the request when stopping the server
pool = Pool(None)
hostport = _split_hostport(listen)
self._http = WSGIServer(hostport,
app,
spawn=pool,
log=logger,
error_log=logger)
self._http.start()
self._addr = self._http.address
logger.info("robot running at %s:%s" % hostport)
if block:
try:
# wait until stop() is called
self._stop_event.wait()
finally:
if self.started:
gevent.Greenlet.spawn(self.stop, timeout=60).join()
def stop(self, timeout=30):
"""
1. stop receiving requests on the REST API
2. wait all currently active request finishes
3. stop all services
4. wait all services stop gracefully
5. serialize all services state to disk
6. exit the process
"""
logger = j.logger.get('zerorobot')
logger.info('stopping robot')
# prevent the signal handler to be called again if
# more signal are received
for h in self._sig_handler:
h.cancel()
logger.info("stop REST API")
logger.info("waiting request to finish")
self._http.stop(timeout=10)
self._addr = None
logger.info("waiting for services to stop")
pool = Pool(30)
def stop_service(service):
try:
service._gracefull_stop(timeout=timeout)
except Exception as err:
logger.warning(
'exception raised while waiting %s %s (%s) to finish: %s',
service.template_uid.name, service.name, service.guid, err)
pool.map(stop_service, scol.list_services())
# here no more requests are comming in, we can safely save all services
self._save_services()
# notify we can exist the process
self._stop_event.set()
def _save_services(self):
"""
serialize all the services on disk
"""
for service in scol.list_services():
# stop all the greenlets attached to the services
service.gl_mgr.stop_all()
service.save()
