def create_app(handle, instance):
check.inst_param(handle, 'handle', ExecutionTargetHandle)
check.inst_param(instance, 'instance', DagsterInstance)
app = Flask('dagster-ui')
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
execution_manager = MultiprocessingExecutionManager()
print('Loading repository...')
context = DagsterGraphQLContext(handle=handle,
instance=instance,
execution_manager=execution_manager,
version=__version__)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
sockets.add_url_rule(
'/graphql', 'graphql',
dagster_graphql_subscription_view(subscription_server, context))
app.add_url_rule(
# should match the `build_local_download_url`
'/download/<string:run_id>/<string:step_key>/<string:file_type>',
'download_view',
download_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
app.add_url_rule('/dagit/notebook', 'notebook',
lambda: notebook_view(request.args))
app.add_url_rule('/static/<path:path>/<string:file>', 'static_view',
static_view)
app.add_url_rule('/vendor/<path:path>/<string:file>', 'vendor_view',
vendor_view)
app.add_url_rule('/<path:_path>', 'index_catchall', index_view)
app.add_url_rule('/', 'index', index_view, defaults={'_path': ''})
CORS(app)
return app
def create_app(handle,
pipeline_run_storage,
use_synchronous_execution_manager=False):
check.inst_param(handle, 'handle', ExecutionTargetHandle)
check.inst_param(pipeline_run_storage, 'pipeline_run_storage',
PipelineRunStorage)
check.bool_param(use_synchronous_execution_manager,
'use_synchronous_execution_manager')
app = Flask('dagster-ui')
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
if use_synchronous_execution_manager:
execution_manager = SynchronousExecutionManager()
else:
execution_manager = MultiprocessingExecutionManager()
context = DagsterGraphQLContext(
handle=handle,
pipeline_runs=pipeline_run_storage,
execution_manager=execution_manager,
version=__version__,
)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
sockets.add_url_rule(
'/graphql', 'graphql',
dagster_graphql_subscription_view(subscription_server, context))
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic glabl request args dict so that notebook_view is testable
app.add_url_rule('/dagit/notebook', 'notebook',
lambda: notebook_view(request.args))
app.add_url_rule('/static/<path:path>/<string:file>', 'static_view',
static_view)
app.add_url_rule('/<path:_path>', 'index_catchall', index_view)
app.add_url_rule('/', 'index', index_view, defaults={'_path': ''})
CORS(app)
return app
def instantiate_app_with_views(context):
app = Flask(
'dagster-ui',
static_url_path='',
static_folder=os.path.join(os.path.dirname(__file__),
'./webapp/build'),
)
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
app.add_url_rule('/graphiql', 'graphiql',
lambda: redirect('/graphql', 301))
sockets.add_url_rule(
'/graphql', 'graphql',
dagster_graphql_subscription_view(subscription_server, context))
app.add_url_rule(
# should match the `build_local_download_url`
'/download/<string:run_id>/<string:step_key>/<string:file_type>',
'download_view',
download_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
app.add_url_rule('/dagit/notebook', 'notebook',
lambda: notebook_view(request.args))
app.add_url_rule('/dagit_info', 'sanity_view', info_view)
app.register_error_handler(404, index_view)
CORS(app)
return app
def create_app(repository_container, pipeline_runs, use_synchronous_execution_manager=False):
app = Flask('dagster-ui')
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
if use_synchronous_execution_manager:
execution_manager = SynchronousExecutionManager()
else:
execution_manager = MultiprocessingExecutionManager()
context = DagsterGraphQLContext(
repository_container=repository_container,
pipeline_runs=pipeline_runs,
execution_manager=execution_manager,
version=__version__,
)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
sockets.add_url_rule(
'/graphql', 'graphql', dagster_graphql_subscription_view(subscription_server, context)
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
app.add_url_rule('/dagit/notebook', 'notebook', notebook_view)
app.add_url_rule('/static/<path:path>/<string:file>', 'static_view', static_view)
app.add_url_rule('/<path:_path>', 'index_catchall', index_view)
app.add_url_rule('/', 'index', index_view, defaults={'_path': ''})
CORS(app)
return app
def __init__(self, config):
self.redis = None
self.backend = None
if config['server'].get('redis_url'):
self.redis = redis.from_url(config['server']['redis_url'])
self.redis_chan = config['server']['redis_chan']
self.backend = ServerBackend(self.redis, self.redis_chan)
else:
print(
'No redis configured, disabling Websockets and remote web console'
)
self.flask_host = config['server']['host']
self.flask_port = config['server']['port']
self.flask_app = Flask(__name__)
self.flask_app.add_url_rule('/', 'index', self._index)
sockets = Sockets(self.flask_app)
# sockets.add_url_rule('/submit', 'submit', self._inbox)
sockets.add_url_rule('/status', 'status', self._status)
self.console = PushingConsole(
self.redis, self.redis_chan,
config['server']['terse']) if self.redis else None
def instantiate_app_with_views(context,
schema,
app_path_prefix,
target_dir=os.path.dirname(__file__)):
app = Flask(
"dagster-ui",
static_url_path=app_path_prefix,
static_folder=os.path.join(target_dir, "./webapp/build"),
)
subscription_server = DagsterSubscriptionServer(schema=schema)
# Websocket routes
sockets = Sockets(app)
sockets.add_url_rule(
f"{app_path_prefix}/graphql",
"graphql",
dagster_graphql_subscription_view(subscription_server, context),
)
# HTTP routes
bp = Blueprint("routes", __name__, url_prefix=app_path_prefix)
bp.add_url_rule("/graphiql", "graphiql",
lambda: redirect(f"{app_path_prefix}/graphql", 301))
bp.add_url_rule(
"/graphql",
"graphql",
DagsterGraphQLView.as_view(
"graphql",
schema=schema,
graphiql=True,
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
bp.add_url_rule(
# should match the `build_local_download_url`
"/download/<string:run_id>/<string:step_key>/<string:file_type>",
"download_view",
download_log_view(context),
)
bp.add_url_rule(
"/download_debug/<string:run_id>",
"download_dump_view",
download_dump_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
bp.add_url_rule("/dagit/notebook", "notebook",
lambda: notebook_view(request.args))
bp.add_url_rule("/dagit_info", "sanity_view", info_view)
index_path = os.path.join(target_dir, "./webapp/build/index.html")
def index_view():
try:
with open(index_path) as f:
rendered_template = render_template_string(f.read())
return rendered_template.replace(
'src="/static', f'src="{app_path_prefix}/static').replace(
'href="/static', f'href="{app_path_prefix}/static')
except FileNotFoundError:
raise Exception(
"""Can't find webapp files. Probably webapp isn't built. If you are using
dagit, then probably it's a corrupted installation or a bug. However, if you are
developing dagit locally, your problem can be fixed as follows:
cd ./python_modules/
make rebuild_dagit""")
def error_redirect(_path):
return index_view()
bp.add_url_rule("/", "index_view", index_view)
bp.context_processor(lambda: {"app_path_prefix": app_path_prefix})
app.app_protocol = lambda environ_path_info: "graphql-ws"
app.register_blueprint(bp)
app.register_error_handler(404, error_redirect)
# if the user asked for a path prefix, handle the naked domain just in case they are not
# filtering inbound traffic elsewhere and redirect to the path prefix.
if app_path_prefix:
app.add_url_rule("/", "force-path-prefix",
lambda: redirect(app_path_prefix, 301))
CORS(app)
return app
def create_app(handle, instance, reloader=None):
check.inst_param(handle, 'handle', ExecutionTargetHandle)
check.inst_param(instance, 'instance', DagsterInstance)
check.opt_inst_param(reloader, 'reloader', Reloader)
app = Flask('dagster-ui')
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
execution_manager_settings = instance.dagit_settings.get(
'execution_manager')
if execution_manager_settings and execution_manager_settings.get(
'max_concurrent_runs'):
execution_manager = QueueingSubprocessExecutionManager(
instance, execution_manager_settings.get('max_concurrent_runs'))
else:
execution_manager = SubprocessExecutionManager(instance)
warn_if_compute_logs_disabled()
print('Loading repository...')
context = DagsterGraphQLContext(
handle=handle,
instance=instance,
execution_manager=execution_manager,
reloader=reloader,
version=__version__,
)
# Automatically initialize scheduler everytime Dagit loads
scheduler_handle = context.scheduler_handle
scheduler = instance.scheduler
if scheduler_handle:
if scheduler:
handle = context.get_handle()
python_path = sys.executable
repository_path = handle.data.repository_yaml
repository = context.get_repository()
scheduler_handle.up(python_path,
repository_path,
repository=repository,
instance=instance)
else:
warnings.warn(MISSING_SCHEDULER_WARNING)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
sockets.add_url_rule(
'/graphql', 'graphql',
dagster_graphql_subscription_view(subscription_server, context))
app.add_url_rule(
# should match the `build_local_download_url`
'/download/<string:run_id>/<string:step_key>/<string:file_type>',
'download_view',
download_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
app.add_url_rule('/dagit/notebook', 'notebook',
lambda: notebook_view(request.args))
app.add_url_rule('/static/<path:path>/<string:file>', 'static_view',
static_view)
app.add_url_rule('/vendor/<path:path>/<string:file>', 'vendor_view',
vendor_view)
app.add_url_rule('/<string:worker_name>.worker.js', 'worker_view',
worker_view)
app.add_url_rule('/dagit_info', 'sanity_view', info_view)
app.add_url_rule('/<path:_path>', 'index_catchall', index_view)
app.add_url_rule('/', 'index', index_view, defaults={'_path': ''})
CORS(app)
return app
def instantiate_app_with_views(context, app_path_prefix):
app = Flask(
'dagster-ui',
static_url_path=app_path_prefix,
static_folder=os.path.join(os.path.dirname(__file__), './webapp/build'),
)
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
# Websocket routes
sockets = Sockets(app)
sockets.add_url_rule(
'{}/graphql'.format(app_path_prefix),
'graphql',
dagster_graphql_subscription_view(subscription_server, context),
)
# HTTP routes
bp = Blueprint('routes', __name__, url_prefix=app_path_prefix)
bp.add_url_rule(
'/graphiql', 'graphiql', lambda: redirect('{}/graphql'.format(app_path_prefix), 301)
)
bp.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
graphiql_template=PLAYGROUND_TEMPLATE.replace('APP_PATH_PREFIX', app_path_prefix),
executor=Executor(),
context=context,
),
)
bp.add_url_rule(
# should match the `build_local_download_url`
'/download/<string:run_id>/<string:step_key>/<string:file_type>',
'download_view',
download_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
bp.add_url_rule('/dagit/notebook', 'notebook', lambda: notebook_view(request.args))
bp.add_url_rule('/dagit_info', 'sanity_view', info_view)
index_path = os.path.join(os.path.dirname(__file__), './webapp/build/index.html')
def index_view(_path):
try:
with open(index_path) as f:
return (
f.read()
.replace('href="/', 'href="{}/'.format(app_path_prefix))
.replace('src="/', 'src="{}/'.format(app_path_prefix))
.replace(
'<meta name="dagit-path-prefix"',
'<meta name="dagit-path-prefix" content="{}"'.format(app_path_prefix),
)
)
except seven.FileNotFoundError:
raise Exception(
'''Can't find webapp files. Probably webapp isn't built. If you are using
dagit, then probably it's a corrupted installation or a bug. However, if you are
developing dagit locally, your problem can be fixed as follows:
cd ./python_modules/
make rebuild_dagit'''
)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
app.register_blueprint(bp)
app.register_error_handler(404, index_view)
# if the user asked for a path prefix, handle the naked domain just in case they are not
# filtering inbound traffic elsewhere and redirect to the path prefix.
if app_path_prefix:
app.add_url_rule('/', 'force-path-prefix', lambda: redirect(app_path_prefix, 301))
CORS(app)
return app
def instantiate_app_with_views(
context: IWorkspaceProcessContext,
schema,
app_path_prefix,
target_dir=os.path.dirname(__file__),
graphql_middleware=None,
include_notebook_route=False,
) -> Flask:
app = Flask(
"dagster-ui",
static_url_path=app_path_prefix,
static_folder=os.path.join(target_dir, "./webapp/build"),
)
subscription_server = DagsterSubscriptionServer(schema=schema)
# Websocket routes
sockets = Sockets(app)
sockets.add_url_rule(
f"{app_path_prefix}/graphql",
"graphql",
dagster_graphql_subscription_view(subscription_server, context),
)
# HTTP routes
bp = Blueprint("routes", __name__, url_prefix=app_path_prefix)
bp.add_url_rule("/graphiql", "graphiql",
lambda: redirect(f"{app_path_prefix}/graphql", 301))
bp.add_url_rule(
"/graphql",
"graphql",
DagsterGraphQLView.as_view(
"graphql",
schema=schema,
graphiql=True,
graphiql_template=PLAYGROUND_TEMPLATE,
context=context,
middleware=graphql_middleware,
),
)
bp.add_url_rule(
# should match the `build_local_download_url`
"/download/<string:run_id>/<string:step_key>/<string:file_type>",
"download_view",
download_log_view(context),
)
bp.add_url_rule(
"/download_debug/<string:run_id>",
"download_dump_view",
download_dump_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
if include_notebook_route:
bp.add_url_rule("/dagit/notebook", "notebook",
lambda: notebook_view(context, request.args))
bp.add_url_rule("/dagit_info", "sanity_view", info_view)
index_path = os.path.join(target_dir, "./webapp/build/index.html")
telemetry_enabled = is_dagit_telemetry_enabled(context.instance)
def index_view(*args, **kwargs): # pylint: disable=unused-argument
try:
with open(index_path) as f:
rendered_template = render_template_string(f.read())
return (rendered_template.replace(
'href="/', f'href="{app_path_prefix}/').replace(
'src="/', f'src="{app_path_prefix}/').replace(
"__PATH_PREFIX__", app_path_prefix).replace(
'"__TELEMETRY_ENABLED__"',
str(telemetry_enabled).lower()).replace(
"NONCE-PLACEHOLDER",
uuid.uuid4().hex))
except FileNotFoundError:
raise Exception(
"""Can't find webapp files. Probably webapp isn't built. If you are using
dagit, then probably it's a corrupted installation or a bug. However, if you are
developing dagit locally, your problem can be fixed as follows:
cd ./python_modules/
make rebuild_dagit""")
bp.add_url_rule("/", "index_view", index_view)
bp.add_url_rule("/<path:path>", "catch_all", index_view)
bp.context_processor(lambda: {"app_path_prefix": app_path_prefix})
app.app_protocol = lambda environ_path_info: "graphql-ws"
app.before_request(initialize_counts)
app.register_blueprint(bp)
app.register_error_handler(404, index_view)
app.after_request(return_counts)
CORS(app)
return app
class ActiveWebService(ServiceBase):
"""
See object_database.frontends.object_database_webtest.py for example
useage.
"""
def __init__(self, db, serviceObject, serviceRuntimeConfig):
ServiceBase.__init__(self, db, serviceObject, serviceRuntimeConfig)
self._logger = logging.getLogger(__name__)
@staticmethod
def configure(db, serviceObject, hostname, port, level_name="INFO"):
db.subscribeToType(Configuration)
with db.transaction():
c = Configuration.lookupAny(service=serviceObject)
if not c:
c = Configuration(service=serviceObject)
c.hostname = hostname
c.port = port
c.log_level = logging.getLevelName(level_name)
@staticmethod
def setLoginPlugin(db,
serviceObject,
loginPluginFactory,
authPlugins,
codebase=None,
config=None):
db.subscribeToType(Configuration)
db.subscribeToType(LoginPlugin)
config = config or {}
with db.transaction():
c = Configuration.lookupAny(service=serviceObject)
if not c:
c = Configuration(service=serviceObject)
login_plugin = LoginPlugin(name="an auth plugin",
login_plugin_factory=loginPluginFactory,
auth_plugins=TupleOf(
OneOf(None,
AuthPluginBase))(authPlugins),
codebase=codebase,
config=config)
c.login_plugin = login_plugin
@staticmethod
def configureFromCommandline(db, serviceObject, args):
"""
Subclasses should take the remaining args from the commandline and
configure using them.
"""
db.subscribeToType(Configuration)
parser = argparse.ArgumentParser("Configure a webservice")
parser.add_argument("--hostname", type=str)
parser.add_argument("--port", type=int)
# optional arguments
parser.add_argument("--log-level",
type=str,
required=False,
default="INFO")
parser.add_argument("--ldap-hostname", type=str, required=False)
parser.add_argument("--ldap-base-dn", type=str, required=False)
parser.add_argument("--ldap-ntlm-domain", type=str, required=False)
parser.add_argument("--authorized-groups",
type=str,
required=False,
nargs="+")
parser.add_argument("--company-name", type=str, required=False)
parsedArgs = parser.parse_args(args)
with db.transaction():
c = Configuration.lookupAny(service=serviceObject)
if not c:
c = Configuration(service=serviceObject)
level_name = parsedArgs.log_level.upper()
level_name = validateLogLevel(level_name, fallback='INFO')
c.port = parsedArgs.port
c.hostname = parsedArgs.hostname
c.log_level = logging.getLevelName(level_name)
if parsedArgs.ldap_base_dn is not None:
ActiveWebService.setLoginPlugin(
db,
serviceObject,
LoginIpPlugin, [
LdapAuthPlugin(parsedArgs.ldap_hostname,
parsedArgs.ldap_base_dn,
parsedArgs.ldap_ntlm_domain,
parsedArgs.authorized_groups)
],
config={'company_name': parsedArgs.company_name})
def initialize(self):
# dict from session id (cookie really) to a a list of
# [cells.SessionState]
self.sessionStates = {}
self.db.subscribeToType(Configuration)
self.db.subscribeToType(LoginPlugin)
self.db.subscribeToSchema(service_schema)
with self.db.transaction():
self.app = Flask(__name__)
CORS(self.app)
self.sockets = Sockets(self.app)
self.configureApp()
self.login_manager = LoginManager(self.app)
self.login_manager.login_view = 'login'
def doWork(self, shouldStop):
self._logger.info("Configuring ActiveWebService")
with self.db.view() as view:
config = Configuration.lookupAny(service=self.serviceObject)
assert config, "No configuration available."
self._logger.setLevel(config.log_level)
host, port = config.hostname, config.port
login_config = config.login_plugin
codebase = login_config.codebase
if codebase is None:
ser_ctx = TypedPythonCodebase.coreSerializationContext()
else:
ser_ctx = codebase.instantiate().serializationContext
view.setSerializationContext(ser_ctx)
self.login_plugin = login_config.login_plugin_factory(
self.db, login_config.auth_plugins, login_config.config)
# register `load_user` method with login_manager
self.login_plugin.load_user = self.login_manager.user_loader(
self.login_plugin.load_user)
self.authorized_groups_text = self.login_plugin.authorized_groups_text
self.login_plugin.init_app(self.app)
self._logger.info("ActiveWebService listening on %s:%s", host, port)
server = pywsgi.WSGIServer((host, port),
self.app,
handler_class=WebSocketHandler)
server.serve_forever()
def configureApp(self):
self.app.config['SECRET_KEY'] = os.environ.get(
'SECRET_KEY') or genToken()
self.app.add_url_rule('/',
endpoint='index',
view_func=lambda: redirect("/services"))
self.app.add_url_rule('/content/<path:path>',
endpoint=None,
view_func=self.sendContent)
self.app.add_url_rule('/services',
endpoint=None,
view_func=self.sendPage)
self.app.add_url_rule('/services/<path:path>',
endpoint=None,
view_func=self.sendPage)
self.app.add_url_rule('/status', view_func=self.statusPage)
self.sockets.add_url_rule('/socket/<path:path>', None, self.mainSocket)
def statusPage(self):
return make_response(jsonify("STATUS: service is up"))
@login_required
def sendPage(self, path=None):
self._logger.info("Sending 'page.html'")
return self.sendContent("page.html")
def displayForPathAndQueryArgs(self, path, queryArgs):
display, toggles = displayAndHeadersForPathAndQueryArgs(
path, queryArgs)
return mainBar(display, toggles, current_user.username,
self.authorized_groups_text)
@login_required
def mainSocket(self, ws, path):
path = str(path).split("/")
queryArgs = dict(request.args.items())
sessionId = request.cookies.get("session")
# wait for the other socket to close if we were bounced
sleep(.25)
sessionState = self._getSessionState(sessionId)
self._logger.info("entering websocket with path %s", path)
reader = None
isFirstMessage = True
# set up message tracking
timestamps = []
lastDumpTimestamp = time.time()
lastDumpMessages = 0
lastDumpFrames = 0
lastDumpTimeSpentCalculating = 0.0
# set up cells
cells = Cells(self.db)
# reset the session state. There's only one per cells (which is why
# we keep a list of sessions.)
sessionState._reset(cells)
cells = cells.withRoot(
Subscribed(
lambda: self.displayForPathAndQueryArgs(path, queryArgs)),
serialization_context=self.db.serializationContext,
session_state=sessionState)
# large messages (more than frames_per_ack frames) send an ack
# after every frames_per_ackth message
largeMessageAck = gevent.queue.Queue()
reader = Greenlet.spawn(
functools.partial(readThread, ws, cells, largeMessageAck,
self._logger))
self._logger.info("Starting main websocket handler with %s", ws)
while not ws.closed:
t0 = time.time()
try:
# make sure user is authenticated
user = self.login_plugin.load_user(current_user.username)
if not user.is_authenticated:
ws.close()
return
messages = cells.renderMessages()
lastDumpTimeSpentCalculating += time.time() - t0
if isFirstMessage:
self._logger.info("Completed first rendering loop")
isFirstMessage = False
for message in messages:
gevent.socket.wait_write(ws.stream.handler.socket.fileno())
writeJsonMessage(message, ws, largeMessageAck,
self._logger)
lastDumpMessages += 1
lastDumpFrames += 1
# log slow messages
if time.time() - lastDumpTimestamp > 60.0:
self._logger.info(
"In the last %.2f seconds, spent %.2f seconds"
" calculating %s messages over %s frames",
time.time() - lastDumpTimestamp,
lastDumpTimeSpentCalculating, lastDumpMessages,
lastDumpFrames)
lastDumpFrames = 0
lastDumpMessages = 0
lastDumpTimeSpentCalculating = 0
lastDumpTimestamp = time.time()
# tell the browser to execute the postscripts that its built up
writeJsonMessage("postscripts", ws, largeMessageAck,
self._logger)
# request an ACK from the browser before sending any more data
# otherwise it can get overloaded and crash because it can't keep
# up with the data volume
writeJsonMessage("request_ack", ws, largeMessageAck,
self._logger)
ack = largeMessageAck.get()
if ack is StopIteration:
raise Exception("Websocket closed.")
cells.wait()
timestamps.append(time.time())
if len(timestamps) > MAX_FPS:
timestamps = timestamps[-MAX_FPS + 1:]
if (time.time() - timestamps[0]) < 1.0:
sleep(1.0 / MAX_FPS + .001)
except Exception:
self._logger.error("Websocket handler error: %s",
traceback.format_exc())
self.sessionStates[sessionId].append(sessionState)
self._logger.info(
"Returning session state to pool for %s. Have %s",
sessionId, len(self.sessionStates[sessionId]))
if reader:
reader.join()
def _getSessionState(self, sessionId):
if sessionId is None:
sessionState = SessionState()
else:
# we keep sessions in a list. This is not great, but if you
# bounce your browser, you'll get the session state you just dropped.
# if you have several windows open, close a few, and then reopen
# you'll get a random one
sessionStateList = self.sessionStates.setdefault(sessionId, [])
if not sessionStateList:
self._logger.info("Creating a new SessionState for %s",
sessionId)
sessionState = SessionState()
else:
sessionState = sessionStateList.pop()
return sessionState
@login_required
def echoSocket(self, ws):
while not ws.closed:
message = ws.receive()
if message is not None:
ws.send(message)
@login_required
def sendContent(self, path):
own_dir = os.path.dirname(__file__)
return send_from_directory(os.path.join(own_dir, "content"), path)
@staticmethod
def serviceDisplay(serviceObject,
instance=None,
objType=None,
queryArgs=None):
c = Configuration.lookupAny(service=serviceObject)
return Card(Text("Host: " + c.hostname) + Text("Port: " + str(c.port)))
class WebAgent:
def __init__(self, appque, prefs):
self.name = 'WebAgent'
self.prefs = prefs
self.log = logging.getLogger("WebAgent")
self.appque = appque
self.orientation = True
self.active = False
self.max_plies = 6
self.display_cache = ""
self.last_cursor_up = 0
self.move_cache = ""
self.info_cache = ""
self.info_provider = {}
self.max_mpv = 1
self.last_board = None
self.last_attribs = None
self.last_pgn = None
self.port = 8001
self.socket_moves = []
self.figrep = {
"int": [1, 2, 3, 4, 5, 6, 0, -1, -2, -3, -4, -5, -6],
"pythc": [(chess.PAWN, chess.WHITE), (chess.KNIGHT, chess.WHITE),
(chess.BISHOP, chess.WHITE), (chess.ROOK, chess.WHITE),
(chess.QUEEN, chess.WHITE), (chess.KING, chess.WHITE),
(chess.PAWN, chess.BLACK), (chess.KNIGHT, chess.BLACK),
(chess.BISHOP, chess.BLACK), (chess.ROOK, chess.BLACK),
(chess.QUEEN, chess.BLACK), (chess.KING, chess.BLACK)],
"unic":
"♟♞♝♜♛♚ ♙♘♗♖♕♔",
"ascii":
"PNBRQK.pnbrqk"
}
self.chesssym = {
"unic": ["-", "×", "†", "‡", "½"],
"ascii": ["-", "x", "+", "#", "1/2"]
}
disable_web_logs = True
if disable_web_logs is True:
wlog = logging.getLogger('werkzeug')
wlog.setLevel(logging.ERROR)
slog = logging.getLogger('geventwebsocket.handler')
slog.setLevel(logging.ERROR)
self.app = Flask(__name__, static_folder='web')
# self.app.config['ENV'] = "MChess_Agent"
self.app.config['SECRET_KEY'] = 'somesecret' # TODO: Investigate
self.app.debug = False
self.app.use_reloader = False
self.sockets = Sockets(self.app)
self.app.add_url_rule('/node_modules/<path:path>', 'node_modules',
self.node_modules)
self.app.add_url_rule('/', 'root', self.web_root)
self.app.add_url_rule('/favicon.ico', 'favicon', self.web_favicon)
self.app.add_url_rule('/index.html', 'index', self.web_root)
self.app.add_url_rule('/scripts/mchess.js', 'script',
self.mchess_script)
self.app.add_url_rule('/styles/mchess.css', 'style', self.mchess_style)
self.app.add_url_rule('/images/turquoise.png', 'logo',
self.mchess_logo)
self.active = True
self.sockets.add_url_rule('/ws', 'ws', self.ws_sockets)
self.ws_clients = {}
self.ws_handle = 0
self.socket_handler() # Start threads for web and ws:sockets
def node_modules(self, path):
# print("NODESTUFF")
return send_from_directory('web/node_modules', path)
def web_root(self):
return self.app.send_static_file('index.html')
def web_favicon(self):
return self.app.send_static_file('favicon.ico')
def ws_dispatch(self, ws, message):
self.log.debug("Client ws_dispatch: ws:{} msg:{}".format(ws, message))
try:
self.appque.put(json.loads(message))
except Exception as e:
self.log.debug("WebClient sent invalid JSON: {}".format(e))
def ws_sockets(self, ws):
self.ws_handle += 1
handle = self.ws_handle
if self.last_board is not None and self.last_attribs is not None:
msg = {
'fen': self.last_board.fen(),
'pgn': self.last_pgn,
'attribs': self.last_attribs
}
try:
ws.send(json.dumps(msg))
except Exception as e:
self.log.warning(
"Sending to WebSocket client {} failed with {}".format(
w, e))
return
self.ws_clients[handle] = ws
while not ws.closed:
message = ws.receive()
self.ws_dispatch(handle, message)
del self.ws_clients[handle]
def mchess_script(self):
return self.app.send_static_file('scripts/mchess.js')
def mchess_style(self):
return self.app.send_static_file('styles/mchess.css')
def mchess_logo(self):
return self.app.send_static_file('images/turquoise.png')
# def sock_connect(self):
# print("CONNECT")
# def sock_message(self, message):
# print("RECEIVED: {}".format(message))
def agent_ready(self):
return self.active
def quit(self):
self.socket_thread_active = False
def display_board(
self,
board,
attribs={
'unicode': True,
'invert': False,
'white_name': 'white',
'black_name': 'black'
}):
self.last_board = board
self.last_attribs = attribs
try:
game = chess.pgn.Game().from_board(board)
game.headers["White"] = attribs["white_name"]
game.headers["Black"] = attribs["black_name"]
pgntxt = str(game)
except Exception as e:
self.log.error("Invalid PGN position, {}".format(e))
return
self.last_pgn = pgntxt
# print("pgn: {}".format(pgntxt))
msg = {'fen': board.fen(), 'pgn': pgntxt, 'attribs': attribs}
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(msg))
except Exception as e:
self.log.warning(
"Sending to WebSocket client {} failed with {}".format(
w, e))
def display_move(self, move_msg):
pass
def display_info(self, board, info):
ninfo = copy.deepcopy(info)
nboard = copy.deepcopy(board)
if 'variant' in ninfo:
ml = []
if nboard.turn is False:
mv = (nboard.fullmove_number, )
mv += ("..", )
for move in ninfo['variant']:
if move is None:
self.log.error("None-move in variant: {}".format(ninfo))
if nboard.turn is True:
mv = (nboard.fullmove_number, )
try:
san = nboard.san(move)
except Exception as e:
self.log.warning(
"Internal error '{}' at san conversion.".format(e))
san = None
if san is not None:
mv += (san, )
else:
self.log.info(
"Variant cut off due to san-conversion-error: '{}'".
format(mv))
break
if nboard.turn is False:
ml.append(mv)
mv = ""
nboard.push(move)
if mv != "":
ml.append(mv)
mv = ""
ninfo['variant'] = ml
msg = {'fenref': nboard.fen(), 'info': ninfo}
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(msg))
except Exception as e:
self.log.warning(
"Sending to WebSocket client {} failed with {}".format(
w, e))
def agent_states(self, msg):
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(msg))
except Exception as e:
self.log.warning(
"Sending to WebSocket client {} failed with {}".format(
w, e))
def set_valid_moves(self, board, vals):
self.socket_moves = []
if vals != None:
for v in vals:
self.socket_moves.append(vals[v])
def socket_event_worker_thread(self, appque, log, app, WebSocketHandler):
server = pywsgi.WSGIServer(('0.0.0.0', self.port),
app,
handler_class=WebSocketHandler)
print("Web browser: http://{}:{}".format(socket.gethostname(),
self.port))
server.serve_forever()
def socket_handler(self):
self.socket_thread_active = True
self.socket_event_thread = threading.Thread(
target=self.socket_event_worker_thread,
args=(self.appque, self.log, self.app, WebSocketHandler))
self.socket_event_thread.setDaemon(True)
self.socket_event_thread.start()
class WebAgent:
def __init__(self, appque, prefs):
self.name = 'WebAgent'
self.prefs = prefs
self.log = logging.getLogger("WebAgent")
self.appque = appque
self.orientation = True
self.active = False
self.max_plies = 6
self.display_cache = ""
self.last_cursor_up = 0
self.move_cache = ""
self.info_cache = ""
self.info_provider = {}
self.agent_state_cache = {}
self.uci_engines_cache = {}
self.display_move_cache = {}
self.valid_moves_cache = {}
self.game_stats_cache = {}
self.max_mpv = 1
self.last_board = None
self.last_attribs = None
self.last_pgn = None
if 'port' in self.prefs:
self.port = self.prefs['port']
else:
self.port = 8001
self.log.warning(f'Port not configured, defaulting to {self.port}')
if 'bind_address' in self.prefs:
self.bind_address = self.prefs['bind_address']
else:
self.bind_address = 'localhost'
self.log.warning(
f'Bind_address not configured, defaulting to f{self.bind_address}, set to "0.0.0.0" for remote accessibility'
)
self.private_key = None
self.public_key = None
if 'tls' in self.prefs and self.prefs['tls'] is True:
if 'private_key' not in self.prefs or 'public_key' not in self.prefs:
self.log.error(
f"Cannot configure tls without public_key and private_key configured!"
)
else:
self.private_key = prefs['private_key']
self.public_key = prefs['public_key']
self.figrep = {
"int": [1, 2, 3, 4, 5, 6, 0, -1, -2, -3, -4, -5, -6],
"pythc": [(chess.PAWN, chess.WHITE), (chess.KNIGHT, chess.WHITE),
(chess.BISHOP, chess.WHITE), (chess.ROOK, chess.WHITE),
(chess.QUEEN, chess.WHITE), (chess.KING, chess.WHITE),
(chess.PAWN, chess.BLACK), (chess.KNIGHT, chess.BLACK),
(chess.BISHOP, chess.BLACK), (chess.ROOK, chess.BLACK),
(chess.QUEEN, chess.BLACK), (chess.KING, chess.BLACK)],
"unic":
"♟♞♝♜♛♚ ♙♘♗♖♕♔",
"ascii":
"PNBRQK.pnbrqk"
}
self.chesssym = {
"unic": ["-", "×", "†", "‡", "½"],
"ascii": ["-", "x", "+", "#", "1/2"]
}
disable_web_logs = True
if disable_web_logs is True:
wlog = logging.getLogger('werkzeug')
wlog.setLevel(logging.ERROR)
slog = logging.getLogger('geventwebsocket.handler')
slog.setLevel(logging.ERROR)
self.app = Flask(__name__, static_folder='web')
# self.app.config['ENV'] = "MChess_Agent"
self.app.config['SECRET_KEY'] = 'secretsauce'
self.app.debug = False
self.app.use_reloader = False
self.sockets = Sockets(self.app)
self.app.add_url_rule('/node_modules/<path:path>', 'node_modules',
self.node_modules)
self.app.add_url_rule('/', 'root', self.web_root)
self.app.add_url_rule('/favicon.ico', 'favicon', self.web_favicon)
self.app.add_url_rule('/index.html', 'index', self.web_root)
self.app.add_url_rule('/scripts/mchess.js', 'script',
self.mchess_script)
self.app.add_url_rule('/styles/mchess.css', 'style', self.mchess_style)
self.app.add_url_rule('/images/<path:path>', 'images', self.images)
self.active = True
self.sockets.add_url_rule('/ws', 'ws', self.ws_sockets)
self.ws_clients = {}
self.ws_handle = 0
self.log.debug("Initializing web server...")
self.socket_handler() # Start threads for web and ws:sockets
def node_modules(self, path):
# print("NODESTUFF")
return send_from_directory('web/node_modules', path)
def web_root(self):
return self.app.send_static_file('index.html')
def web_favicon(self):
return self.app.send_static_file('favicon.ico')
def ws_dispatch(self, ws, message):
# self.log.info(f"message received: {message}")
if message is not None:
self.log.debug("Client ws_dispatch: ws:{} msg:{}".format(
ws, message))
try:
self.log.info(f"Received: {message}")
self.appque.put(json.loads(message))
except Exception as e:
self.log.debug("WebClient sent invalid JSON: {}".format(e))
def ws_sockets(self, ws):
self.ws_handle += 1
handle = self.ws_handle
if self.last_board is not None and self.last_attribs is not None:
msg = {
'cmd': 'display_board',
'fen': self.last_board.fen(),
'pgn': self.last_pgn,
'attribs': self.last_attribs
}
try:
ws.send(json.dumps(msg))
except Exception as e:
self.log.warning(
"Sending to WebSocket client {} failed with {}".format(
handle, e))
return
for actor in self.agent_state_cache:
msg = self.agent_state_cache[actor]
try:
ws.send(json.dumps(msg))
except Exception as e:
self.log.warning(
f"Failed to update agents states to new web-socket client: {e}"
)
if self.uci_engines_cache != {}:
ws.send(json.dumps(self.uci_engines_cache))
if self.display_move_cache != {}:
ws.send(json.dumps(self.display_move_cache))
if self.valid_moves_cache != {}:
ws.send(json.dumps(self.valid_moves_cache))
if self.game_stats_cache != {}:
ws.send(json.dumps(self.game_stats_cache))
self.ws_clients[handle] = ws
while not ws.closed:
message = ws.receive()
self.ws_dispatch(handle, message)
del self.ws_clients[handle]
def mchess_script(self):
return self.app.send_static_file('scripts/mchess.js')
def mchess_style(self):
return self.app.send_static_file('styles/mchess.css')
def images(self, path):
return send_from_directory('web/images', path)
# def sock_connect(self):
# print("CONNECT")
# def sock_message(self, message):
# print("RECEIVED: {}".format(message))
def agent_ready(self):
return self.active
def quit(self):
self.socket_thread_active = False
def display_board(
self,
board,
attribs={
'unicode': True,
'invert': False,
'white_name': 'white',
'black_name': 'black'
}):
self.last_board = board
self.last_attribs = attribs
try:
game = chess.pgn.Game().from_board(board)
game.headers["White"] = attribs["white_name"]
game.headers["Black"] = attribs["black_name"]
pgntxt = str(game)
except Exception as e:
self.log.error("Invalid PGN position, {}".format(e))
return
self.last_pgn = pgntxt
# print("pgn: {}".format(pgntxt))
msg = {
'cmd': 'display_board',
'fen': board.fen(),
'pgn': pgntxt,
'attribs': attribs
}
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(msg))
except Exception as e:
self.log.warning(
"Sending board to WebSocket client {} failed with {}".
format(w, e))
def display_move(self, move_msg):
self.display_move_cache = move_msg
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(move_msg))
except Exception as e:
self.log.warning(
"Sending display_move to WebSocket client {} failed with {}"
.format(w, e))
def set_valid_moves(self, board, vals):
self.log.info("web set valid called.")
self.valid_moves_cache = {
"cmd": "valid_moves",
"valid_moves": [],
'actor': 'WebAgent'
}
if vals != None:
for v in vals:
self.valid_moves_cache['valid_moves'].append(vals[v])
self.log.info(f"Valid-moves: {self.valid_moves_cache}")
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(self.valid_moves_cache))
except Exception as e:
self.log.warning(
"Sending display_move to WebSocket client {} failed with {}"
.format(w, e))
def display_info(self, board, info):
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(info))
except Exception as e:
self.log.warning(
"Sending move-info to WebSocket client {} failed with {}".
format(w, e))
def engine_list(self, msg):
for engine in msg["engines"]:
self.log.info(f"Engine {engine} announced.")
self.uci_engines_cache = msg
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(msg))
except Exception as e:
self.log.warning(
"Sending uci-info to WebSocket client {} failed with {}".
format(w, e))
def game_stats(self, stats):
msg = {'cmd': 'game_stats', 'stats': stats, 'actor': 'WebAgent'}
self.game_stats_cache = msg
self.log.info(f"Game stats: {msg}")
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(msg))
except Exception as e:
self.log.warning(
"Sending game_stats to WebSocket client {} failed with {}".
format(w, e))
def agent_states(self, msg):
self.agent_state_cache[msg['actor']] = msg
for w in self.ws_clients:
try:
self.ws_clients[w].send(json.dumps(msg))
except Exception as e:
self.log.warning(
"Sending agent-state info to WebSocket client {} failed with {}"
.format(w, e))
# def set_valid_moves(self, board, vals):
# self.socket_moves = []
# if vals != None:
# for v in vals:
# self.socket_moves.append(vals[v])
def socket_event_worker_thread(self, appque, log, app, WebSocketHandler):
if self.bind_address == '0.0.0.0':
address = socket.gethostname()
else:
address = self.bind_address
if self.private_key is None or self.public_key is None:
server = pywsgi.WSGIServer((self.bind_address, self.port),
app,
handler_class=WebSocketHandler)
protocol = 'http'
self.log.info(f"Web browser: {protocol}://{address}:{self.port}")
else:
server = pywsgi.WSGIServer((self.bind_address, self.port),
app,
keyfile=self.private_key,
certfile=self.public_key,
handler_class=WebSocketHandler)
protocol = 'https'
self.log.info(f"Web browser: {protocol}://{address}:{self.port}")
print(f"Web browser: {protocol}://{address}:{self.port}")
server.serve_forever()
def socket_handler(self):
self.socket_thread_active = True
self.socket_event_thread = threading.Thread(
target=self.socket_event_worker_thread,
args=(self.appque, self.log, self.app, WebSocketHandler))
self.socket_event_thread.setDaemon(True)
self.socket_event_thread.start()
class ActiveWebService(ServiceBase):
def __init__(self, db, serviceObject, serviceRuntimeConfig):
ServiceBase.__init__(self, db, serviceObject, serviceRuntimeConfig)
self._logger = logging.getLogger(__name__)
@staticmethod
def initialize_auth_plugin(auth_type, authorized_groups=None, hostname=None, ldap_base_dn=None, ldap_ntlm_domain=None):
if auth_type == "NONE":
return None
elif auth_type == "PERMISSIVE":
return PermissiveAuthPlugin()
elif auth_type == "FORBIDDEN":
return AuthPluginBase()
elif auth_type == "LDAP":
if not hostname:
raise Exception("Missing required argument for LDAP: --hostname")
if not ldap_base_dn:
raise Exception("Missing required argument for LDAP: --base-dn")
return LdapAuthPlugin(
hostname=hostname,
base_dn=ldap_base_dn,
ntlm_domain=ldap_ntlm_domain,
authorized_groups=authorized_groups
)
else:
raise Exception("Unkown auth-type: {}".format(auth_type))
@staticmethod
def configureFromCommandline(db, serviceObject, args):
"""Subclasses should take the remaining args from the commandline and configure using them"""
db.subscribeToType(Configuration)
with db.transaction():
c = Configuration.lookupAny(service=serviceObject)
if not c:
c = Configuration(service=serviceObject)
parser = argparse.ArgumentParser("Configure a webservice")
parser.add_argument("--hostname", type=str)
parser.add_argument("--port", type=int)
parser.add_argument("--log-level", type=str, required=False, default="INFO")
parser.add_argument("--auth", type=str, required=False, default="LDAP")
parser.add_argument("--authorized-groups", nargs='+', type=str, required=False, default=())
parser.add_argument("--auth-hostname", type=str, required=False, default="")
parser.add_argument("--ldap-base-dn", type=str, required=False, default="")
parser.add_argument("--ldap-ntlm-domain", type=str, required=False, default="")
parser.add_argument("--company-name", type=str, required=False, default="")
parsedArgs = parser.parse_args(args)
VALID_AUTH_TYPES = ["NONE", "LDAP", "PERMISSIVE", "FORBIDDEN"]
auth_type = parsedArgs.auth.upper()
if auth_type not in VALID_AUTH_TYPES:
raise Exception("invalid --auth value: {auth}. Must be one of {options}"
.format(auth=parsedArgs.auth, options=VALID_AUTH_TYPES))
level_name = parsedArgs.log_level.upper()
checkLogLevelValidity(level_name)
c.port = parsedArgs.port
c.hostname = parsedArgs.hostname
c.log_level = logging.getLevelName(level_name)
c.auth_type = auth_type
c.authorized_groups = TupleOf(str)(parsedArgs.authorized_groups)
c.auth_hostname = parsedArgs.auth_hostname
c.ldap_base_dn = parsedArgs.ldap_base_dn
c.ldap_ntlm_domain = parsedArgs.ldap_ntlm_domain
c.company_name = parsedArgs.company_name
def initialize(self):
self.db.subscribeToType(Configuration)
self.db.subscribeToType(User)
self.db.subscribeToSchema(service_schema)
with self.db.transaction():
self.app = Flask(__name__)
CORS(self.app)
self.sockets = Sockets(self.app)
self.configureApp()
self.login_manager = LoginManager(self.app)
self.load_user = self.login_manager.user_loader(self.load_user)
self.login_manager.login_view = 'login'
def load_user(self, username):
with self.db.view():
return UserWrapper.makeFromUser(User.lookupAny(username=username))
def doWork(self, shouldStop):
self._logger.info("Configuring ActiveWebService")
with self.db.view():
config = Configuration.lookupAny(service=self.serviceObject)
assert config, "No configuration available."
self._logger.setLevel(config.log_level)
host, port = config.hostname, config.port
self.authorized_groups = config.authorized_groups
self.authorized_groups_text = "All"
if self.authorized_groups:
self.authorized_groups_text = ", ".join(self.authorized_groups)
self.auth_plugin = self.initialize_auth_plugin(
config.auth_type,
authorized_groups=config.authorized_groups,
hostname=config.auth_hostname,
ldap_base_dn=config.ldap_base_dn,
ldap_ntlm_domain=config.ldap_ntlm_domain
)
self.company_name = config.company_name
self._logger.info("ActiveWebService listening on %s:%s", host, port)
server = pywsgi.WSGIServer((host, port), self.app, handler_class=WebSocketHandler)
server.serve_forever()
def configureApp(self):
instanceName = self.serviceObject.name
self.app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY') or genToken()
self.app.add_url_rule('/', endpoint=None, view_func=lambda: redirect("/services"))
self.app.add_url_rule('/content/<path:path>', endpoint=None, view_func=self.sendContent)
self.app.add_url_rule('/services', endpoint=None, view_func=self.sendPage)
self.app.add_url_rule('/services/<path:path>', endpoint=None, view_func=self.sendPage)
self.app.add_url_rule('/login', endpoint=None, view_func=self.login, methods=['GET', 'POST'])
self.app.add_url_rule('/logout', endpoint=None, view_func=self.logout)
self.sockets.add_url_rule('/socket/<path:path>', None, self.mainSocket)
@revisionConflictRetry
def login_user(self, username):
with self.db.transaction():
users = User.lookupAll(username=username)
if len(users) == 0:
user = User(username=username)
elif len(users) == 1:
user = users[0]
elif len(users) > 1:
raise Exception("multiple users found with username={}".format(username))
else:
raise Exception("This should never happen: len(users)={}".format(len(users)))
user.login()
login_user(UserWrapper.makeFromUser(user))
def login(self):
if current_user.is_authenticated:
return redirect('/')
if self.auth_plugin is None:
self.login_user('anonymous')
return redirect('/')
form = LoginForm()
if form.validate_on_submit():
username = form.username.data
password = form.password.data
if not self.auth_plugin.authenticate(username, password):
flash(
'Invalid username or password or not a member of an '
'authorized group. Please try again.',
'danger')
return render_template(
'login.html',
form=form,
title=self.company_name,
authorized_groups_text=self.authorized_groups_text
)
self.login_user(username)
return redirect('/')
if form.errors:
flash(form.errors, 'danger')
return render_template(
'login.html',
form=form,
title=self.company_name,
authorized_groups_text=self.authorized_groups_text
)
def logout(self):
logout_user()
return redirect('/')
@login_required
def sendPage(self, path=None):
return self.sendContent("page.html")
def mainDisplay(self):
def serviceCountSetter(service, ct):
def f():
service.target_count = ct
return f
serviceCounts = list(range(5)) + list(range(10,100,10)) + list(range(100,400,25)) + list(range(400,1001,100))
buttons = Sequence([
Padding(),
Button(
Sequence([Octicon('shield').color('green'), Span('Lock ALL')]),
lambda: [s.lock() for s in service_schema.Service.lookupAll()]),
Button(
Sequence([Octicon('shield').color('orange'), Span('Prepare ALL')]),
lambda: [s.prepare() for s in service_schema.Service.lookupAll()]),
Button(
Sequence([Octicon('stop').color('red'), Span('Unlock ALL')]),
lambda: [s.unlock() for s in service_schema.Service.lookupAll()]),
])
tabs = Tabs(
Services=Table(
colFun=lambda: [
'Service', 'Codebase Status', 'Codebase', 'Module', 'Class',
'Placement', 'Active', 'TargetCount', 'Cores', 'RAM', 'Boot Status'],
rowFun=lambda:
sorted(service_schema.Service.lookupAll(), key=lambda s:s.name),
headerFun=lambda x: x,
rendererFun=lambda s, field: Subscribed(lambda:
Clickable(s.name, "/services/" + s.name) if field == 'Service' else
( Clickable(Sequence([Octicon('stop').color('red'), Span('Unlocked')]),
lambda: s.lock()) if s.isUnlocked else
Clickable(Sequence([Octicon('shield').color('green'), Span('Locked')]),
lambda: s.prepare()) if s.isLocked else
Clickable(Sequence([Octicon('shield').color('orange'), Span('Prepared')]),
lambda: s.unlock())) if field == 'Codebase Status' else
(str(s.codebase) if s.codebase else "") if field == 'Codebase' else
s.service_module_name if field == 'Module' else
s.service_class_name if field == 'Class' else
s.placement if field == 'Placement' else
Subscribed(lambda: len(service_schema.ServiceInstance.lookupAll(service=s))) if field == 'Active' else
Dropdown(s.target_count, [(str(ct), serviceCountSetter(s, ct)) for ct in serviceCounts])
if field == 'TargetCount' else
str(s.coresUsed) if field == 'Cores' else
str(s.gbRamUsed) if field == 'RAM' else
(Popover(Octicon("alert"), "Failed", Traceback(s.lastFailureReason or "<Unknown>")) if s.isThrottled() else "") if field == 'Boot Status' else
""
),
maxRowsPerPage=50
),
Hosts=Table(
colFun=lambda: ['Connection', 'IsMaster', 'Hostname', 'RAM ALLOCATION', 'CORE ALLOCATION', 'SERVICE COUNT', 'CPU USE', 'RAM USE'],
rowFun=lambda: sorted(service_schema.ServiceHost.lookupAll(), key=lambda s:s.hostname),
headerFun=lambda x: x,
rendererFun=lambda s,field: Subscribed(lambda:
s.connection._identity if field == "Connection" else
str(s.isMaster) if field == "IsMaster" else
s.hostname if field == "Hostname" else
"%.1f / %.1f" % (s.gbRamUsed, s.maxGbRam) if field == "RAM ALLOCATION" else
"%s / %s" % (s.coresUsed, s.maxCores) if field == "CORE ALLOCATION" else
str(len(service_schema.ServiceInstance.lookupAll(host=s))) if field == "SERVICE COUNT" else
"%2.1f" % (s.cpuUse * 100) + "%" if field == "CPU USE" else
("%2.1f" % s.actualMemoryUseGB) + " GB" if field == "RAM USE" else
""
),
maxRowsPerPage=50
)
)
return Sequence([buttons, tabs])
def pathToDisplay(self, path, queryArgs):
if len(path) and path[0] == 'services':
if len(path) == 1:
return self.mainDisplay()
serviceObj = service_schema.Service.lookupAny(name=path[1])
if serviceObj is None:
return Traceback("Unknown service %s" % path[1])
serviceType = serviceObj.instantiateServiceType()
if len(path) == 2:
return (
Subscribed(lambda: serviceType.serviceDisplay(serviceObj, queryArgs=queryArgs))
.withSerializationContext(serviceObj.getSerializationContext())
)
typename = path[2]
schemas = serviceObj.findModuleSchemas()
typeObj = None
for s in schemas:
typeObj = s.lookupFullyQualifiedTypeByName(typename)
if typeObj:
break
if typeObj is None:
return Traceback("Can't find fully-qualified type %s" % typename)
if len(path) == 3:
return (
serviceType.serviceDisplay(serviceObj, objType=typename, queryArgs=queryArgs)
.withSerializationContext(serviceObj.getSerializationContext())
)
instance = typeObj.fromIdentity(path[3])
return (
serviceType.serviceDisplay(serviceObj, instance=instance, queryArgs=queryArgs)
.withSerializationContext(serviceObj.getSerializationContext())
)
return Traceback("Invalid url path: %s" % path)
def addMainBar(self, display):
with self.db.view():
current_username = current_user.username
return (
HeaderBar(
[Subscribed(lambda:
Dropdown(
"Service",
[("All", "/services")] +
[(s.name, "/services/" + s.name) for
s in sorted(service_schema.Service.lookupAll(), key=lambda s:s.name)]
),
),
Dropdown(
Octicon("three-bars"),
[
(Sequence([Octicon('person'),
Span('Logged in as: {}'.format(current_username))]),
lambda: None),
(Sequence([Octicon('organization'),
Span('Authorized Groups: {}'.format(self.authorized_groups_text))]),
lambda: None),
(Sequence([Octicon('sign-out'),
Span('Logout')]),
'/logout')
])
]) +
Main(display)
)
@login_required
def mainSocket(self, ws, path):
path = str(path).split("/")
queryArgs = dict(request.args)
self._logger.info("path = %s", path)
reader = None
try:
self._logger.info("Starting main websocket handler with %s", ws)
cells = Cells(self.db)
cells.root.setRootSerializationContext(self.db.serializationContext)
cells.root.setChild(self.addMainBar(Subscribed(lambda: self.pathToDisplay(path, queryArgs))))
timestamps = []
lastDumpTimestamp = time.time()
lastDumpMessages = 0
lastDumpFrames = 0
lastDumpTimeSpentCalculating = 0.0
def readThread():
while not ws.closed:
msg = ws.receive()
if msg is None:
return
else:
try:
jsonMsg = json.loads(msg)
cell_id = jsonMsg.get('target_cell')
cell = cells[cell_id]
if cell is not None:
cell.onMessage(jsonMsg)
except Exception:
self._logger.error("Exception in inbound message: %s", traceback.format_exc())
cells.triggerIfHasDirty()
reader = Greenlet.spawn(readThread)
while not ws.closed:
t0 = time.time()
messages = cells.renderMessages()
user = self.load_user(current_user.username)
if not user.is_authenticated:
ws.close()
return
lastDumpTimeSpentCalculating += time.time() - t0
for message in messages:
gevent.socket.wait_write(ws.stream.handler.socket.fileno())
ws.send(json.dumps(message))
lastDumpMessages += 1
lastDumpFrames += 1
if time.time() - lastDumpTimestamp > 5.0:
self._logger.info("In the last %.2f seconds, spent %.2f seconds calculating %s messages over %s frames",
time.time() - lastDumpTimestamp,
lastDumpTimeSpentCalculating,
lastDumpMessages,
lastDumpFrames
)
lastDumpFrames = 0
lastDumpMessages = 0
lastDumpTimeSpentCalculating = 0
lastDumpTimestamp = time.time()
ws.send(json.dumps("postscripts"))
cells.wait()
timestamps.append(time.time())
if len(timestamps) > MAX_FPS:
timestamps = timestamps[-MAX_FPS+1:]
if (time.time() - timestamps[0]) < 1.0:
sleep(1.0 / MAX_FPS + .001)
except Exception:
self._logger.error("Websocket handler error: %s", traceback.format_exc())
finally:
if reader:
reader.join()
@login_required
def echoSocket(self, ws):
while not ws.closed:
message = ws.receive()
if message is not None:
ws.send(message)
@login_required
def sendContent(self, path):
own_dir = os.path.dirname(__file__)
return send_from_directory(os.path.join(own_dir, "content"), path)
@staticmethod
def serviceDisplay(serviceObject, instance=None, objType=None, queryArgs=None):
c = Configuration.lookupAny(service=serviceObject)
return Card(Text("Host: " + c.hostname) + Text("Port: " + str(c.port)))