def create_app(handle, instance): check.inst_param(handle, 'handle', ExecutionTargetHandle) check.inst_param(instance, 'instance', DagsterInstance) app = Flask('dagster-ui') sockets = Sockets(app) app.app_protocol = lambda environ_path_info: 'graphql-ws' schema = create_schema() subscription_server = DagsterSubscriptionServer(schema=schema) execution_manager = MultiprocessingExecutionManager() print('Loading repository...') context = DagsterGraphQLContext(handle=handle, instance=instance, execution_manager=execution_manager, version=__version__) app.add_url_rule( '/graphql', 'graphql', DagsterGraphQLView.as_view( 'graphql', schema=schema, graphiql=True, # XXX(freiksenet): Pass proper ws url graphiql_template=PLAYGROUND_TEMPLATE, executor=Executor(), context=context, ), ) sockets.add_url_rule( '/graphql', 'graphql', dagster_graphql_subscription_view(subscription_server, context)) app.add_url_rule( # should match the `build_local_download_url` '/download/<string:run_id>/<string:step_key>/<string:file_type>', 'download_view', download_view(context), ) # these routes are specifically for the Dagit UI and are not part of the graphql # API that we want other people to consume, so they're separate for now. # Also grabbing the magic global request args dict so that notebook_view is testable app.add_url_rule('/dagit/notebook', 'notebook', lambda: notebook_view(request.args)) app.add_url_rule('/static/<path:path>/<string:file>', 'static_view', static_view) app.add_url_rule('/vendor/<path:path>/<string:file>', 'vendor_view', vendor_view) app.add_url_rule('/<path:_path>', 'index_catchall', index_view) app.add_url_rule('/', 'index', index_view, defaults={'_path': ''}) CORS(app) return app
def create_app(handle, pipeline_run_storage, use_synchronous_execution_manager=False): check.inst_param(handle, 'handle', ExecutionTargetHandle) check.inst_param(pipeline_run_storage, 'pipeline_run_storage', PipelineRunStorage) check.bool_param(use_synchronous_execution_manager, 'use_synchronous_execution_manager') app = Flask('dagster-ui') sockets = Sockets(app) app.app_protocol = lambda environ_path_info: 'graphql-ws' schema = create_schema() subscription_server = DagsterSubscriptionServer(schema=schema) if use_synchronous_execution_manager: execution_manager = SynchronousExecutionManager() else: execution_manager = MultiprocessingExecutionManager() context = DagsterGraphQLContext( handle=handle, pipeline_runs=pipeline_run_storage, execution_manager=execution_manager, version=__version__, ) app.add_url_rule( '/graphql', 'graphql', DagsterGraphQLView.as_view( 'graphql', schema=schema, graphiql=True, # XXX(freiksenet): Pass proper ws url graphiql_template=PLAYGROUND_TEMPLATE, executor=Executor(), context=context, ), ) sockets.add_url_rule( '/graphql', 'graphql', dagster_graphql_subscription_view(subscription_server, context)) # these routes are specifically for the Dagit UI and are not part of the graphql # API that we want other people to consume, so they're separate for now. # Also grabbing the magic glabl request args dict so that notebook_view is testable app.add_url_rule('/dagit/notebook', 'notebook', lambda: notebook_view(request.args)) app.add_url_rule('/static/<path:path>/<string:file>', 'static_view', static_view) app.add_url_rule('/<path:_path>', 'index_catchall', index_view) app.add_url_rule('/', 'index', index_view, defaults={'_path': ''}) CORS(app) return app
def instantiate_app_with_views(context): app = Flask( 'dagster-ui', static_url_path='', static_folder=os.path.join(os.path.dirname(__file__), './webapp/build'), ) sockets = Sockets(app) app.app_protocol = lambda environ_path_info: 'graphql-ws' schema = create_schema() subscription_server = DagsterSubscriptionServer(schema=schema) app.add_url_rule( '/graphql', 'graphql', DagsterGraphQLView.as_view( 'graphql', schema=schema, graphiql=True, # XXX(freiksenet): Pass proper ws url graphiql_template=PLAYGROUND_TEMPLATE, executor=Executor(), context=context, ), ) app.add_url_rule('/graphiql', 'graphiql', lambda: redirect('/graphql', 301)) sockets.add_url_rule( '/graphql', 'graphql', dagster_graphql_subscription_view(subscription_server, context)) app.add_url_rule( # should match the `build_local_download_url` '/download/<string:run_id>/<string:step_key>/<string:file_type>', 'download_view', download_view(context), ) # these routes are specifically for the Dagit UI and are not part of the graphql # API that we want other people to consume, so they're separate for now. # Also grabbing the magic global request args dict so that notebook_view is testable app.add_url_rule('/dagit/notebook', 'notebook', lambda: notebook_view(request.args)) app.add_url_rule('/dagit_info', 'sanity_view', info_view) app.register_error_handler(404, index_view) CORS(app) return app
def create_app(repository_container, pipeline_runs, use_synchronous_execution_manager=False): app = Flask('dagster-ui') sockets = Sockets(app) app.app_protocol = lambda environ_path_info: 'graphql-ws' schema = create_schema() subscription_server = DagsterSubscriptionServer(schema=schema) if use_synchronous_execution_manager: execution_manager = SynchronousExecutionManager() else: execution_manager = MultiprocessingExecutionManager() context = DagsterGraphQLContext( repository_container=repository_container, pipeline_runs=pipeline_runs, execution_manager=execution_manager, version=__version__, ) app.add_url_rule( '/graphql', 'graphql', DagsterGraphQLView.as_view( 'graphql', schema=schema, graphiql=True, # XXX(freiksenet): Pass proper ws url graphiql_template=PLAYGROUND_TEMPLATE, executor=Executor(), context=context, ), ) sockets.add_url_rule( '/graphql', 'graphql', dagster_graphql_subscription_view(subscription_server, context) ) # these routes are specifically for the Dagit UI and are not part of the graphql # API that we want other people to consume, so they're separate for now. app.add_url_rule('/dagit/notebook', 'notebook', notebook_view) app.add_url_rule('/static/<path:path>/<string:file>', 'static_view', static_view) app.add_url_rule('/<path:_path>', 'index_catchall', index_view) app.add_url_rule('/', 'index', index_view, defaults={'_path': ''}) CORS(app) return app
def __init__(self, config): self.redis = None self.backend = None if config['server'].get('redis_url'): self.redis = redis.from_url(config['server']['redis_url']) self.redis_chan = config['server']['redis_chan'] self.backend = ServerBackend(self.redis, self.redis_chan) else: print( 'No redis configured, disabling Websockets and remote web console' ) self.flask_host = config['server']['host'] self.flask_port = config['server']['port'] self.flask_app = Flask(__name__) self.flask_app.add_url_rule('/', 'index', self._index) sockets = Sockets(self.flask_app) # sockets.add_url_rule('/submit', 'submit', self._inbox) sockets.add_url_rule('/status', 'status', self._status) self.console = PushingConsole( self.redis, self.redis_chan, config['server']['terse']) if self.redis else None
def instantiate_app_with_views(context, schema, app_path_prefix, target_dir=os.path.dirname(__file__)): app = Flask( "dagster-ui", static_url_path=app_path_prefix, static_folder=os.path.join(target_dir, "./webapp/build"), ) subscription_server = DagsterSubscriptionServer(schema=schema) # Websocket routes sockets = Sockets(app) sockets.add_url_rule( f"{app_path_prefix}/graphql", "graphql", dagster_graphql_subscription_view(subscription_server, context), ) # HTTP routes bp = Blueprint("routes", __name__, url_prefix=app_path_prefix) bp.add_url_rule("/graphiql", "graphiql", lambda: redirect(f"{app_path_prefix}/graphql", 301)) bp.add_url_rule( "/graphql", "graphql", DagsterGraphQLView.as_view( "graphql", schema=schema, graphiql=True, graphiql_template=PLAYGROUND_TEMPLATE, executor=Executor(), context=context, ), ) bp.add_url_rule( # should match the `build_local_download_url` "/download/<string:run_id>/<string:step_key>/<string:file_type>", "download_view", download_log_view(context), ) bp.add_url_rule( "/download_debug/<string:run_id>", "download_dump_view", download_dump_view(context), ) # these routes are specifically for the Dagit UI and are not part of the graphql # API that we want other people to consume, so they're separate for now. # Also grabbing the magic global request args dict so that notebook_view is testable bp.add_url_rule("/dagit/notebook", "notebook", lambda: notebook_view(request.args)) bp.add_url_rule("/dagit_info", "sanity_view", info_view) index_path = os.path.join(target_dir, "./webapp/build/index.html") def index_view(): try: with open(index_path) as f: rendered_template = render_template_string(f.read()) return rendered_template.replace( 'src="/static', f'src="{app_path_prefix}/static').replace( 'href="/static', f'href="{app_path_prefix}/static') except FileNotFoundError: raise Exception( """Can't find webapp files. Probably webapp isn't built. If you are using dagit, then probably it's a corrupted installation or a bug. However, if you are developing dagit locally, your problem can be fixed as follows: cd ./python_modules/ make rebuild_dagit""") def error_redirect(_path): return index_view() bp.add_url_rule("/", "index_view", index_view) bp.context_processor(lambda: {"app_path_prefix": app_path_prefix}) app.app_protocol = lambda environ_path_info: "graphql-ws" app.register_blueprint(bp) app.register_error_handler(404, error_redirect) # if the user asked for a path prefix, handle the naked domain just in case they are not # filtering inbound traffic elsewhere and redirect to the path prefix. if app_path_prefix: app.add_url_rule("/", "force-path-prefix", lambda: redirect(app_path_prefix, 301)) CORS(app) return app
def create_app(handle, instance, reloader=None): check.inst_param(handle, 'handle', ExecutionTargetHandle) check.inst_param(instance, 'instance', DagsterInstance) check.opt_inst_param(reloader, 'reloader', Reloader) app = Flask('dagster-ui') sockets = Sockets(app) app.app_protocol = lambda environ_path_info: 'graphql-ws' schema = create_schema() subscription_server = DagsterSubscriptionServer(schema=schema) execution_manager_settings = instance.dagit_settings.get( 'execution_manager') if execution_manager_settings and execution_manager_settings.get( 'max_concurrent_runs'): execution_manager = QueueingSubprocessExecutionManager( instance, execution_manager_settings.get('max_concurrent_runs')) else: execution_manager = SubprocessExecutionManager(instance) warn_if_compute_logs_disabled() print('Loading repository...') context = DagsterGraphQLContext( handle=handle, instance=instance, execution_manager=execution_manager, reloader=reloader, version=__version__, ) # Automatically initialize scheduler everytime Dagit loads scheduler_handle = context.scheduler_handle scheduler = instance.scheduler if scheduler_handle: if scheduler: handle = context.get_handle() python_path = sys.executable repository_path = handle.data.repository_yaml repository = context.get_repository() scheduler_handle.up(python_path, repository_path, repository=repository, instance=instance) else: warnings.warn(MISSING_SCHEDULER_WARNING) app.add_url_rule( '/graphql', 'graphql', DagsterGraphQLView.as_view( 'graphql', schema=schema, graphiql=True, # XXX(freiksenet): Pass proper ws url graphiql_template=PLAYGROUND_TEMPLATE, executor=Executor(), context=context, ), ) sockets.add_url_rule( '/graphql', 'graphql', dagster_graphql_subscription_view(subscription_server, context)) app.add_url_rule( # should match the `build_local_download_url` '/download/<string:run_id>/<string:step_key>/<string:file_type>', 'download_view', download_view(context), ) # these routes are specifically for the Dagit UI and are not part of the graphql # API that we want other people to consume, so they're separate for now. # Also grabbing the magic global request args dict so that notebook_view is testable app.add_url_rule('/dagit/notebook', 'notebook', lambda: notebook_view(request.args)) app.add_url_rule('/static/<path:path>/<string:file>', 'static_view', static_view) app.add_url_rule('/vendor/<path:path>/<string:file>', 'vendor_view', vendor_view) app.add_url_rule('/<string:worker_name>.worker.js', 'worker_view', worker_view) app.add_url_rule('/dagit_info', 'sanity_view', info_view) app.add_url_rule('/<path:_path>', 'index_catchall', index_view) app.add_url_rule('/', 'index', index_view, defaults={'_path': ''}) CORS(app) return app
def instantiate_app_with_views(context, app_path_prefix): app = Flask( 'dagster-ui', static_url_path=app_path_prefix, static_folder=os.path.join(os.path.dirname(__file__), './webapp/build'), ) schema = create_schema() subscription_server = DagsterSubscriptionServer(schema=schema) # Websocket routes sockets = Sockets(app) sockets.add_url_rule( '{}/graphql'.format(app_path_prefix), 'graphql', dagster_graphql_subscription_view(subscription_server, context), ) # HTTP routes bp = Blueprint('routes', __name__, url_prefix=app_path_prefix) bp.add_url_rule( '/graphiql', 'graphiql', lambda: redirect('{}/graphql'.format(app_path_prefix), 301) ) bp.add_url_rule( '/graphql', 'graphql', DagsterGraphQLView.as_view( 'graphql', schema=schema, graphiql=True, graphiql_template=PLAYGROUND_TEMPLATE.replace('APP_PATH_PREFIX', app_path_prefix), executor=Executor(), context=context, ), ) bp.add_url_rule( # should match the `build_local_download_url` '/download/<string:run_id>/<string:step_key>/<string:file_type>', 'download_view', download_view(context), ) # these routes are specifically for the Dagit UI and are not part of the graphql # API that we want other people to consume, so they're separate for now. # Also grabbing the magic global request args dict so that notebook_view is testable bp.add_url_rule('/dagit/notebook', 'notebook', lambda: notebook_view(request.args)) bp.add_url_rule('/dagit_info', 'sanity_view', info_view) index_path = os.path.join(os.path.dirname(__file__), './webapp/build/index.html') def index_view(_path): try: with open(index_path) as f: return ( f.read() .replace('href="/', 'href="{}/'.format(app_path_prefix)) .replace('src="/', 'src="{}/'.format(app_path_prefix)) .replace( '<meta name="dagit-path-prefix"', '<meta name="dagit-path-prefix" content="{}"'.format(app_path_prefix), ) ) except seven.FileNotFoundError: raise Exception( '''Can't find webapp files. Probably webapp isn't built. If you are using dagit, then probably it's a corrupted installation or a bug. However, if you are developing dagit locally, your problem can be fixed as follows: cd ./python_modules/ make rebuild_dagit''' ) app.app_protocol = lambda environ_path_info: 'graphql-ws' app.register_blueprint(bp) app.register_error_handler(404, index_view) # if the user asked for a path prefix, handle the naked domain just in case they are not # filtering inbound traffic elsewhere and redirect to the path prefix. if app_path_prefix: app.add_url_rule('/', 'force-path-prefix', lambda: redirect(app_path_prefix, 301)) CORS(app) return app
def instantiate_app_with_views( context: IWorkspaceProcessContext, schema, app_path_prefix, target_dir=os.path.dirname(__file__), graphql_middleware=None, include_notebook_route=False, ) -> Flask: app = Flask( "dagster-ui", static_url_path=app_path_prefix, static_folder=os.path.join(target_dir, "./webapp/build"), ) subscription_server = DagsterSubscriptionServer(schema=schema) # Websocket routes sockets = Sockets(app) sockets.add_url_rule( f"{app_path_prefix}/graphql", "graphql", dagster_graphql_subscription_view(subscription_server, context), ) # HTTP routes bp = Blueprint("routes", __name__, url_prefix=app_path_prefix) bp.add_url_rule("/graphiql", "graphiql", lambda: redirect(f"{app_path_prefix}/graphql", 301)) bp.add_url_rule( "/graphql", "graphql", DagsterGraphQLView.as_view( "graphql", schema=schema, graphiql=True, graphiql_template=PLAYGROUND_TEMPLATE, context=context, middleware=graphql_middleware, ), ) bp.add_url_rule( # should match the `build_local_download_url` "/download/<string:run_id>/<string:step_key>/<string:file_type>", "download_view", download_log_view(context), ) bp.add_url_rule( "/download_debug/<string:run_id>", "download_dump_view", download_dump_view(context), ) # these routes are specifically for the Dagit UI and are not part of the graphql # API that we want other people to consume, so they're separate for now. # Also grabbing the magic global request args dict so that notebook_view is testable if include_notebook_route: bp.add_url_rule("/dagit/notebook", "notebook", lambda: notebook_view(context, request.args)) bp.add_url_rule("/dagit_info", "sanity_view", info_view) index_path = os.path.join(target_dir, "./webapp/build/index.html") telemetry_enabled = is_dagit_telemetry_enabled(context.instance) def index_view(*args, **kwargs): # pylint: disable=unused-argument try: with open(index_path) as f: rendered_template = render_template_string(f.read()) return (rendered_template.replace( 'href="/', f'href="{app_path_prefix}/').replace( 'src="/', f'src="{app_path_prefix}/').replace( "__PATH_PREFIX__", app_path_prefix).replace( '"__TELEMETRY_ENABLED__"', str(telemetry_enabled).lower()).replace( "NONCE-PLACEHOLDER", uuid.uuid4().hex)) except FileNotFoundError: raise Exception( """Can't find webapp files. Probably webapp isn't built. If you are using dagit, then probably it's a corrupted installation or a bug. However, if you are developing dagit locally, your problem can be fixed as follows: cd ./python_modules/ make rebuild_dagit""") bp.add_url_rule("/", "index_view", index_view) bp.add_url_rule("/<path:path>", "catch_all", index_view) bp.context_processor(lambda: {"app_path_prefix": app_path_prefix}) app.app_protocol = lambda environ_path_info: "graphql-ws" app.before_request(initialize_counts) app.register_blueprint(bp) app.register_error_handler(404, index_view) app.after_request(return_counts) CORS(app) return app
class ActiveWebService(ServiceBase): """ See object_database.frontends.object_database_webtest.py for example useage. """ def __init__(self, db, serviceObject, serviceRuntimeConfig): ServiceBase.__init__(self, db, serviceObject, serviceRuntimeConfig) self._logger = logging.getLogger(__name__) @staticmethod def configure(db, serviceObject, hostname, port, level_name="INFO"): db.subscribeToType(Configuration) with db.transaction(): c = Configuration.lookupAny(service=serviceObject) if not c: c = Configuration(service=serviceObject) c.hostname = hostname c.port = port c.log_level = logging.getLevelName(level_name) @staticmethod def setLoginPlugin(db, serviceObject, loginPluginFactory, authPlugins, codebase=None, config=None): db.subscribeToType(Configuration) db.subscribeToType(LoginPlugin) config = config or {} with db.transaction(): c = Configuration.lookupAny(service=serviceObject) if not c: c = Configuration(service=serviceObject) login_plugin = LoginPlugin(name="an auth plugin", login_plugin_factory=loginPluginFactory, auth_plugins=TupleOf( OneOf(None, AuthPluginBase))(authPlugins), codebase=codebase, config=config) c.login_plugin = login_plugin @staticmethod def configureFromCommandline(db, serviceObject, args): """ Subclasses should take the remaining args from the commandline and configure using them. """ db.subscribeToType(Configuration) parser = argparse.ArgumentParser("Configure a webservice") parser.add_argument("--hostname", type=str) parser.add_argument("--port", type=int) # optional arguments parser.add_argument("--log-level", type=str, required=False, default="INFO") parser.add_argument("--ldap-hostname", type=str, required=False) parser.add_argument("--ldap-base-dn", type=str, required=False) parser.add_argument("--ldap-ntlm-domain", type=str, required=False) parser.add_argument("--authorized-groups", type=str, required=False, nargs="+") parser.add_argument("--company-name", type=str, required=False) parsedArgs = parser.parse_args(args) with db.transaction(): c = Configuration.lookupAny(service=serviceObject) if not c: c = Configuration(service=serviceObject) level_name = parsedArgs.log_level.upper() level_name = validateLogLevel(level_name, fallback='INFO') c.port = parsedArgs.port c.hostname = parsedArgs.hostname c.log_level = logging.getLevelName(level_name) if parsedArgs.ldap_base_dn is not None: ActiveWebService.setLoginPlugin( db, serviceObject, LoginIpPlugin, [ LdapAuthPlugin(parsedArgs.ldap_hostname, parsedArgs.ldap_base_dn, parsedArgs.ldap_ntlm_domain, parsedArgs.authorized_groups) ], config={'company_name': parsedArgs.company_name}) def initialize(self): # dict from session id (cookie really) to a a list of # [cells.SessionState] self.sessionStates = {} self.db.subscribeToType(Configuration) self.db.subscribeToType(LoginPlugin) self.db.subscribeToSchema(service_schema) with self.db.transaction(): self.app = Flask(__name__) CORS(self.app) self.sockets = Sockets(self.app) self.configureApp() self.login_manager = LoginManager(self.app) self.login_manager.login_view = 'login' def doWork(self, shouldStop): self._logger.info("Configuring ActiveWebService") with self.db.view() as view: config = Configuration.lookupAny(service=self.serviceObject) assert config, "No configuration available." self._logger.setLevel(config.log_level) host, port = config.hostname, config.port login_config = config.login_plugin codebase = login_config.codebase if codebase is None: ser_ctx = TypedPythonCodebase.coreSerializationContext() else: ser_ctx = codebase.instantiate().serializationContext view.setSerializationContext(ser_ctx) self.login_plugin = login_config.login_plugin_factory( self.db, login_config.auth_plugins, login_config.config) # register `load_user` method with login_manager self.login_plugin.load_user = self.login_manager.user_loader( self.login_plugin.load_user) self.authorized_groups_text = self.login_plugin.authorized_groups_text self.login_plugin.init_app(self.app) self._logger.info("ActiveWebService listening on %s:%s", host, port) server = pywsgi.WSGIServer((host, port), self.app, handler_class=WebSocketHandler) server.serve_forever() def configureApp(self): self.app.config['SECRET_KEY'] = os.environ.get( 'SECRET_KEY') or genToken() self.app.add_url_rule('/', endpoint='index', view_func=lambda: redirect("/services")) self.app.add_url_rule('/content/<path:path>', endpoint=None, view_func=self.sendContent) self.app.add_url_rule('/services', endpoint=None, view_func=self.sendPage) self.app.add_url_rule('/services/<path:path>', endpoint=None, view_func=self.sendPage) self.app.add_url_rule('/status', view_func=self.statusPage) self.sockets.add_url_rule('/socket/<path:path>', None, self.mainSocket) def statusPage(self): return make_response(jsonify("STATUS: service is up")) @login_required def sendPage(self, path=None): self._logger.info("Sending 'page.html'") return self.sendContent("page.html") def displayForPathAndQueryArgs(self, path, queryArgs): display, toggles = displayAndHeadersForPathAndQueryArgs( path, queryArgs) return mainBar(display, toggles, current_user.username, self.authorized_groups_text) @login_required def mainSocket(self, ws, path): path = str(path).split("/") queryArgs = dict(request.args.items()) sessionId = request.cookies.get("session") # wait for the other socket to close if we were bounced sleep(.25) sessionState = self._getSessionState(sessionId) self._logger.info("entering websocket with path %s", path) reader = None isFirstMessage = True # set up message tracking timestamps = [] lastDumpTimestamp = time.time() lastDumpMessages = 0 lastDumpFrames = 0 lastDumpTimeSpentCalculating = 0.0 # set up cells cells = Cells(self.db) # reset the session state. There's only one per cells (which is why # we keep a list of sessions.) sessionState._reset(cells) cells = cells.withRoot( Subscribed( lambda: self.displayForPathAndQueryArgs(path, queryArgs)), serialization_context=self.db.serializationContext, session_state=sessionState) # large messages (more than frames_per_ack frames) send an ack # after every frames_per_ackth message largeMessageAck = gevent.queue.Queue() reader = Greenlet.spawn( functools.partial(readThread, ws, cells, largeMessageAck, self._logger)) self._logger.info("Starting main websocket handler with %s", ws) while not ws.closed: t0 = time.time() try: # make sure user is authenticated user = self.login_plugin.load_user(current_user.username) if not user.is_authenticated: ws.close() return messages = cells.renderMessages() lastDumpTimeSpentCalculating += time.time() - t0 if isFirstMessage: self._logger.info("Completed first rendering loop") isFirstMessage = False for message in messages: gevent.socket.wait_write(ws.stream.handler.socket.fileno()) writeJsonMessage(message, ws, largeMessageAck, self._logger) lastDumpMessages += 1 lastDumpFrames += 1 # log slow messages if time.time() - lastDumpTimestamp > 60.0: self._logger.info( "In the last %.2f seconds, spent %.2f seconds" " calculating %s messages over %s frames", time.time() - lastDumpTimestamp, lastDumpTimeSpentCalculating, lastDumpMessages, lastDumpFrames) lastDumpFrames = 0 lastDumpMessages = 0 lastDumpTimeSpentCalculating = 0 lastDumpTimestamp = time.time() # tell the browser to execute the postscripts that its built up writeJsonMessage("postscripts", ws, largeMessageAck, self._logger) # request an ACK from the browser before sending any more data # otherwise it can get overloaded and crash because it can't keep # up with the data volume writeJsonMessage("request_ack", ws, largeMessageAck, self._logger) ack = largeMessageAck.get() if ack is StopIteration: raise Exception("Websocket closed.") cells.wait() timestamps.append(time.time()) if len(timestamps) > MAX_FPS: timestamps = timestamps[-MAX_FPS + 1:] if (time.time() - timestamps[0]) < 1.0: sleep(1.0 / MAX_FPS + .001) except Exception: self._logger.error("Websocket handler error: %s", traceback.format_exc()) self.sessionStates[sessionId].append(sessionState) self._logger.info( "Returning session state to pool for %s. Have %s", sessionId, len(self.sessionStates[sessionId])) if reader: reader.join() def _getSessionState(self, sessionId): if sessionId is None: sessionState = SessionState() else: # we keep sessions in a list. This is not great, but if you # bounce your browser, you'll get the session state you just dropped. # if you have several windows open, close a few, and then reopen # you'll get a random one sessionStateList = self.sessionStates.setdefault(sessionId, []) if not sessionStateList: self._logger.info("Creating a new SessionState for %s", sessionId) sessionState = SessionState() else: sessionState = sessionStateList.pop() return sessionState @login_required def echoSocket(self, ws): while not ws.closed: message = ws.receive() if message is not None: ws.send(message) @login_required def sendContent(self, path): own_dir = os.path.dirname(__file__) return send_from_directory(os.path.join(own_dir, "content"), path) @staticmethod def serviceDisplay(serviceObject, instance=None, objType=None, queryArgs=None): c = Configuration.lookupAny(service=serviceObject) return Card(Text("Host: " + c.hostname) + Text("Port: " + str(c.port)))
class WebAgent: def __init__(self, appque, prefs): self.name = 'WebAgent' self.prefs = prefs self.log = logging.getLogger("WebAgent") self.appque = appque self.orientation = True self.active = False self.max_plies = 6 self.display_cache = "" self.last_cursor_up = 0 self.move_cache = "" self.info_cache = "" self.info_provider = {} self.max_mpv = 1 self.last_board = None self.last_attribs = None self.last_pgn = None self.port = 8001 self.socket_moves = [] self.figrep = { "int": [1, 2, 3, 4, 5, 6, 0, -1, -2, -3, -4, -5, -6], "pythc": [(chess.PAWN, chess.WHITE), (chess.KNIGHT, chess.WHITE), (chess.BISHOP, chess.WHITE), (chess.ROOK, chess.WHITE), (chess.QUEEN, chess.WHITE), (chess.KING, chess.WHITE), (chess.PAWN, chess.BLACK), (chess.KNIGHT, chess.BLACK), (chess.BISHOP, chess.BLACK), (chess.ROOK, chess.BLACK), (chess.QUEEN, chess.BLACK), (chess.KING, chess.BLACK)], "unic": "♟♞♝♜♛♚ ♙♘♗♖♕♔", "ascii": "PNBRQK.pnbrqk" } self.chesssym = { "unic": ["-", "×", "†", "‡", "½"], "ascii": ["-", "x", "+", "#", "1/2"] } disable_web_logs = True if disable_web_logs is True: wlog = logging.getLogger('werkzeug') wlog.setLevel(logging.ERROR) slog = logging.getLogger('geventwebsocket.handler') slog.setLevel(logging.ERROR) self.app = Flask(__name__, static_folder='web') # self.app.config['ENV'] = "MChess_Agent" self.app.config['SECRET_KEY'] = 'somesecret' # TODO: Investigate self.app.debug = False self.app.use_reloader = False self.sockets = Sockets(self.app) self.app.add_url_rule('/node_modules/<path:path>', 'node_modules', self.node_modules) self.app.add_url_rule('/', 'root', self.web_root) self.app.add_url_rule('/favicon.ico', 'favicon', self.web_favicon) self.app.add_url_rule('/index.html', 'index', self.web_root) self.app.add_url_rule('/scripts/mchess.js', 'script', self.mchess_script) self.app.add_url_rule('/styles/mchess.css', 'style', self.mchess_style) self.app.add_url_rule('/images/turquoise.png', 'logo', self.mchess_logo) self.active = True self.sockets.add_url_rule('/ws', 'ws', self.ws_sockets) self.ws_clients = {} self.ws_handle = 0 self.socket_handler() # Start threads for web and ws:sockets def node_modules(self, path): # print("NODESTUFF") return send_from_directory('web/node_modules', path) def web_root(self): return self.app.send_static_file('index.html') def web_favicon(self): return self.app.send_static_file('favicon.ico') def ws_dispatch(self, ws, message): self.log.debug("Client ws_dispatch: ws:{} msg:{}".format(ws, message)) try: self.appque.put(json.loads(message)) except Exception as e: self.log.debug("WebClient sent invalid JSON: {}".format(e)) def ws_sockets(self, ws): self.ws_handle += 1 handle = self.ws_handle if self.last_board is not None and self.last_attribs is not None: msg = { 'fen': self.last_board.fen(), 'pgn': self.last_pgn, 'attribs': self.last_attribs } try: ws.send(json.dumps(msg)) except Exception as e: self.log.warning( "Sending to WebSocket client {} failed with {}".format( w, e)) return self.ws_clients[handle] = ws while not ws.closed: message = ws.receive() self.ws_dispatch(handle, message) del self.ws_clients[handle] def mchess_script(self): return self.app.send_static_file('scripts/mchess.js') def mchess_style(self): return self.app.send_static_file('styles/mchess.css') def mchess_logo(self): return self.app.send_static_file('images/turquoise.png') # def sock_connect(self): # print("CONNECT") # def sock_message(self, message): # print("RECEIVED: {}".format(message)) def agent_ready(self): return self.active def quit(self): self.socket_thread_active = False def display_board( self, board, attribs={ 'unicode': True, 'invert': False, 'white_name': 'white', 'black_name': 'black' }): self.last_board = board self.last_attribs = attribs try: game = chess.pgn.Game().from_board(board) game.headers["White"] = attribs["white_name"] game.headers["Black"] = attribs["black_name"] pgntxt = str(game) except Exception as e: self.log.error("Invalid PGN position, {}".format(e)) return self.last_pgn = pgntxt # print("pgn: {}".format(pgntxt)) msg = {'fen': board.fen(), 'pgn': pgntxt, 'attribs': attribs} for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(msg)) except Exception as e: self.log.warning( "Sending to WebSocket client {} failed with {}".format( w, e)) def display_move(self, move_msg): pass def display_info(self, board, info): ninfo = copy.deepcopy(info) nboard = copy.deepcopy(board) if 'variant' in ninfo: ml = [] if nboard.turn is False: mv = (nboard.fullmove_number, ) mv += ("..", ) for move in ninfo['variant']: if move is None: self.log.error("None-move in variant: {}".format(ninfo)) if nboard.turn is True: mv = (nboard.fullmove_number, ) try: san = nboard.san(move) except Exception as e: self.log.warning( "Internal error '{}' at san conversion.".format(e)) san = None if san is not None: mv += (san, ) else: self.log.info( "Variant cut off due to san-conversion-error: '{}'". format(mv)) break if nboard.turn is False: ml.append(mv) mv = "" nboard.push(move) if mv != "": ml.append(mv) mv = "" ninfo['variant'] = ml msg = {'fenref': nboard.fen(), 'info': ninfo} for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(msg)) except Exception as e: self.log.warning( "Sending to WebSocket client {} failed with {}".format( w, e)) def agent_states(self, msg): for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(msg)) except Exception as e: self.log.warning( "Sending to WebSocket client {} failed with {}".format( w, e)) def set_valid_moves(self, board, vals): self.socket_moves = [] if vals != None: for v in vals: self.socket_moves.append(vals[v]) def socket_event_worker_thread(self, appque, log, app, WebSocketHandler): server = pywsgi.WSGIServer(('0.0.0.0', self.port), app, handler_class=WebSocketHandler) print("Web browser: http://{}:{}".format(socket.gethostname(), self.port)) server.serve_forever() def socket_handler(self): self.socket_thread_active = True self.socket_event_thread = threading.Thread( target=self.socket_event_worker_thread, args=(self.appque, self.log, self.app, WebSocketHandler)) self.socket_event_thread.setDaemon(True) self.socket_event_thread.start()
class WebAgent: def __init__(self, appque, prefs): self.name = 'WebAgent' self.prefs = prefs self.log = logging.getLogger("WebAgent") self.appque = appque self.orientation = True self.active = False self.max_plies = 6 self.display_cache = "" self.last_cursor_up = 0 self.move_cache = "" self.info_cache = "" self.info_provider = {} self.agent_state_cache = {} self.uci_engines_cache = {} self.display_move_cache = {} self.valid_moves_cache = {} self.game_stats_cache = {} self.max_mpv = 1 self.last_board = None self.last_attribs = None self.last_pgn = None if 'port' in self.prefs: self.port = self.prefs['port'] else: self.port = 8001 self.log.warning(f'Port not configured, defaulting to {self.port}') if 'bind_address' in self.prefs: self.bind_address = self.prefs['bind_address'] else: self.bind_address = 'localhost' self.log.warning( f'Bind_address not configured, defaulting to f{self.bind_address}, set to "0.0.0.0" for remote accessibility' ) self.private_key = None self.public_key = None if 'tls' in self.prefs and self.prefs['tls'] is True: if 'private_key' not in self.prefs or 'public_key' not in self.prefs: self.log.error( f"Cannot configure tls without public_key and private_key configured!" ) else: self.private_key = prefs['private_key'] self.public_key = prefs['public_key'] self.figrep = { "int": [1, 2, 3, 4, 5, 6, 0, -1, -2, -3, -4, -5, -6], "pythc": [(chess.PAWN, chess.WHITE), (chess.KNIGHT, chess.WHITE), (chess.BISHOP, chess.WHITE), (chess.ROOK, chess.WHITE), (chess.QUEEN, chess.WHITE), (chess.KING, chess.WHITE), (chess.PAWN, chess.BLACK), (chess.KNIGHT, chess.BLACK), (chess.BISHOP, chess.BLACK), (chess.ROOK, chess.BLACK), (chess.QUEEN, chess.BLACK), (chess.KING, chess.BLACK)], "unic": "♟♞♝♜♛♚ ♙♘♗♖♕♔", "ascii": "PNBRQK.pnbrqk" } self.chesssym = { "unic": ["-", "×", "†", "‡", "½"], "ascii": ["-", "x", "+", "#", "1/2"] } disable_web_logs = True if disable_web_logs is True: wlog = logging.getLogger('werkzeug') wlog.setLevel(logging.ERROR) slog = logging.getLogger('geventwebsocket.handler') slog.setLevel(logging.ERROR) self.app = Flask(__name__, static_folder='web') # self.app.config['ENV'] = "MChess_Agent" self.app.config['SECRET_KEY'] = 'secretsauce' self.app.debug = False self.app.use_reloader = False self.sockets = Sockets(self.app) self.app.add_url_rule('/node_modules/<path:path>', 'node_modules', self.node_modules) self.app.add_url_rule('/', 'root', self.web_root) self.app.add_url_rule('/favicon.ico', 'favicon', self.web_favicon) self.app.add_url_rule('/index.html', 'index', self.web_root) self.app.add_url_rule('/scripts/mchess.js', 'script', self.mchess_script) self.app.add_url_rule('/styles/mchess.css', 'style', self.mchess_style) self.app.add_url_rule('/images/<path:path>', 'images', self.images) self.active = True self.sockets.add_url_rule('/ws', 'ws', self.ws_sockets) self.ws_clients = {} self.ws_handle = 0 self.log.debug("Initializing web server...") self.socket_handler() # Start threads for web and ws:sockets def node_modules(self, path): # print("NODESTUFF") return send_from_directory('web/node_modules', path) def web_root(self): return self.app.send_static_file('index.html') def web_favicon(self): return self.app.send_static_file('favicon.ico') def ws_dispatch(self, ws, message): # self.log.info(f"message received: {message}") if message is not None: self.log.debug("Client ws_dispatch: ws:{} msg:{}".format( ws, message)) try: self.log.info(f"Received: {message}") self.appque.put(json.loads(message)) except Exception as e: self.log.debug("WebClient sent invalid JSON: {}".format(e)) def ws_sockets(self, ws): self.ws_handle += 1 handle = self.ws_handle if self.last_board is not None and self.last_attribs is not None: msg = { 'cmd': 'display_board', 'fen': self.last_board.fen(), 'pgn': self.last_pgn, 'attribs': self.last_attribs } try: ws.send(json.dumps(msg)) except Exception as e: self.log.warning( "Sending to WebSocket client {} failed with {}".format( handle, e)) return for actor in self.agent_state_cache: msg = self.agent_state_cache[actor] try: ws.send(json.dumps(msg)) except Exception as e: self.log.warning( f"Failed to update agents states to new web-socket client: {e}" ) if self.uci_engines_cache != {}: ws.send(json.dumps(self.uci_engines_cache)) if self.display_move_cache != {}: ws.send(json.dumps(self.display_move_cache)) if self.valid_moves_cache != {}: ws.send(json.dumps(self.valid_moves_cache)) if self.game_stats_cache != {}: ws.send(json.dumps(self.game_stats_cache)) self.ws_clients[handle] = ws while not ws.closed: message = ws.receive() self.ws_dispatch(handle, message) del self.ws_clients[handle] def mchess_script(self): return self.app.send_static_file('scripts/mchess.js') def mchess_style(self): return self.app.send_static_file('styles/mchess.css') def images(self, path): return send_from_directory('web/images', path) # def sock_connect(self): # print("CONNECT") # def sock_message(self, message): # print("RECEIVED: {}".format(message)) def agent_ready(self): return self.active def quit(self): self.socket_thread_active = False def display_board( self, board, attribs={ 'unicode': True, 'invert': False, 'white_name': 'white', 'black_name': 'black' }): self.last_board = board self.last_attribs = attribs try: game = chess.pgn.Game().from_board(board) game.headers["White"] = attribs["white_name"] game.headers["Black"] = attribs["black_name"] pgntxt = str(game) except Exception as e: self.log.error("Invalid PGN position, {}".format(e)) return self.last_pgn = pgntxt # print("pgn: {}".format(pgntxt)) msg = { 'cmd': 'display_board', 'fen': board.fen(), 'pgn': pgntxt, 'attribs': attribs } for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(msg)) except Exception as e: self.log.warning( "Sending board to WebSocket client {} failed with {}". format(w, e)) def display_move(self, move_msg): self.display_move_cache = move_msg for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(move_msg)) except Exception as e: self.log.warning( "Sending display_move to WebSocket client {} failed with {}" .format(w, e)) def set_valid_moves(self, board, vals): self.log.info("web set valid called.") self.valid_moves_cache = { "cmd": "valid_moves", "valid_moves": [], 'actor': 'WebAgent' } if vals != None: for v in vals: self.valid_moves_cache['valid_moves'].append(vals[v]) self.log.info(f"Valid-moves: {self.valid_moves_cache}") for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(self.valid_moves_cache)) except Exception as e: self.log.warning( "Sending display_move to WebSocket client {} failed with {}" .format(w, e)) def display_info(self, board, info): for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(info)) except Exception as e: self.log.warning( "Sending move-info to WebSocket client {} failed with {}". format(w, e)) def engine_list(self, msg): for engine in msg["engines"]: self.log.info(f"Engine {engine} announced.") self.uci_engines_cache = msg for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(msg)) except Exception as e: self.log.warning( "Sending uci-info to WebSocket client {} failed with {}". format(w, e)) def game_stats(self, stats): msg = {'cmd': 'game_stats', 'stats': stats, 'actor': 'WebAgent'} self.game_stats_cache = msg self.log.info(f"Game stats: {msg}") for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(msg)) except Exception as e: self.log.warning( "Sending game_stats to WebSocket client {} failed with {}". format(w, e)) def agent_states(self, msg): self.agent_state_cache[msg['actor']] = msg for w in self.ws_clients: try: self.ws_clients[w].send(json.dumps(msg)) except Exception as e: self.log.warning( "Sending agent-state info to WebSocket client {} failed with {}" .format(w, e)) # def set_valid_moves(self, board, vals): # self.socket_moves = [] # if vals != None: # for v in vals: # self.socket_moves.append(vals[v]) def socket_event_worker_thread(self, appque, log, app, WebSocketHandler): if self.bind_address == '0.0.0.0': address = socket.gethostname() else: address = self.bind_address if self.private_key is None or self.public_key is None: server = pywsgi.WSGIServer((self.bind_address, self.port), app, handler_class=WebSocketHandler) protocol = 'http' self.log.info(f"Web browser: {protocol}://{address}:{self.port}") else: server = pywsgi.WSGIServer((self.bind_address, self.port), app, keyfile=self.private_key, certfile=self.public_key, handler_class=WebSocketHandler) protocol = 'https' self.log.info(f"Web browser: {protocol}://{address}:{self.port}") print(f"Web browser: {protocol}://{address}:{self.port}") server.serve_forever() def socket_handler(self): self.socket_thread_active = True self.socket_event_thread = threading.Thread( target=self.socket_event_worker_thread, args=(self.appque, self.log, self.app, WebSocketHandler)) self.socket_event_thread.setDaemon(True) self.socket_event_thread.start()
class ActiveWebService(ServiceBase): def __init__(self, db, serviceObject, serviceRuntimeConfig): ServiceBase.__init__(self, db, serviceObject, serviceRuntimeConfig) self._logger = logging.getLogger(__name__) @staticmethod def initialize_auth_plugin(auth_type, authorized_groups=None, hostname=None, ldap_base_dn=None, ldap_ntlm_domain=None): if auth_type == "NONE": return None elif auth_type == "PERMISSIVE": return PermissiveAuthPlugin() elif auth_type == "FORBIDDEN": return AuthPluginBase() elif auth_type == "LDAP": if not hostname: raise Exception("Missing required argument for LDAP: --hostname") if not ldap_base_dn: raise Exception("Missing required argument for LDAP: --base-dn") return LdapAuthPlugin( hostname=hostname, base_dn=ldap_base_dn, ntlm_domain=ldap_ntlm_domain, authorized_groups=authorized_groups ) else: raise Exception("Unkown auth-type: {}".format(auth_type)) @staticmethod def configureFromCommandline(db, serviceObject, args): """Subclasses should take the remaining args from the commandline and configure using them""" db.subscribeToType(Configuration) with db.transaction(): c = Configuration.lookupAny(service=serviceObject) if not c: c = Configuration(service=serviceObject) parser = argparse.ArgumentParser("Configure a webservice") parser.add_argument("--hostname", type=str) parser.add_argument("--port", type=int) parser.add_argument("--log-level", type=str, required=False, default="INFO") parser.add_argument("--auth", type=str, required=False, default="LDAP") parser.add_argument("--authorized-groups", nargs='+', type=str, required=False, default=()) parser.add_argument("--auth-hostname", type=str, required=False, default="") parser.add_argument("--ldap-base-dn", type=str, required=False, default="") parser.add_argument("--ldap-ntlm-domain", type=str, required=False, default="") parser.add_argument("--company-name", type=str, required=False, default="") parsedArgs = parser.parse_args(args) VALID_AUTH_TYPES = ["NONE", "LDAP", "PERMISSIVE", "FORBIDDEN"] auth_type = parsedArgs.auth.upper() if auth_type not in VALID_AUTH_TYPES: raise Exception("invalid --auth value: {auth}. Must be one of {options}" .format(auth=parsedArgs.auth, options=VALID_AUTH_TYPES)) level_name = parsedArgs.log_level.upper() checkLogLevelValidity(level_name) c.port = parsedArgs.port c.hostname = parsedArgs.hostname c.log_level = logging.getLevelName(level_name) c.auth_type = auth_type c.authorized_groups = TupleOf(str)(parsedArgs.authorized_groups) c.auth_hostname = parsedArgs.auth_hostname c.ldap_base_dn = parsedArgs.ldap_base_dn c.ldap_ntlm_domain = parsedArgs.ldap_ntlm_domain c.company_name = parsedArgs.company_name def initialize(self): self.db.subscribeToType(Configuration) self.db.subscribeToType(User) self.db.subscribeToSchema(service_schema) with self.db.transaction(): self.app = Flask(__name__) CORS(self.app) self.sockets = Sockets(self.app) self.configureApp() self.login_manager = LoginManager(self.app) self.load_user = self.login_manager.user_loader(self.load_user) self.login_manager.login_view = 'login' def load_user(self, username): with self.db.view(): return UserWrapper.makeFromUser(User.lookupAny(username=username)) def doWork(self, shouldStop): self._logger.info("Configuring ActiveWebService") with self.db.view(): config = Configuration.lookupAny(service=self.serviceObject) assert config, "No configuration available." self._logger.setLevel(config.log_level) host, port = config.hostname, config.port self.authorized_groups = config.authorized_groups self.authorized_groups_text = "All" if self.authorized_groups: self.authorized_groups_text = ", ".join(self.authorized_groups) self.auth_plugin = self.initialize_auth_plugin( config.auth_type, authorized_groups=config.authorized_groups, hostname=config.auth_hostname, ldap_base_dn=config.ldap_base_dn, ldap_ntlm_domain=config.ldap_ntlm_domain ) self.company_name = config.company_name self._logger.info("ActiveWebService listening on %s:%s", host, port) server = pywsgi.WSGIServer((host, port), self.app, handler_class=WebSocketHandler) server.serve_forever() def configureApp(self): instanceName = self.serviceObject.name self.app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY') or genToken() self.app.add_url_rule('/', endpoint=None, view_func=lambda: redirect("/services")) self.app.add_url_rule('/content/<path:path>', endpoint=None, view_func=self.sendContent) self.app.add_url_rule('/services', endpoint=None, view_func=self.sendPage) self.app.add_url_rule('/services/<path:path>', endpoint=None, view_func=self.sendPage) self.app.add_url_rule('/login', endpoint=None, view_func=self.login, methods=['GET', 'POST']) self.app.add_url_rule('/logout', endpoint=None, view_func=self.logout) self.sockets.add_url_rule('/socket/<path:path>', None, self.mainSocket) @revisionConflictRetry def login_user(self, username): with self.db.transaction(): users = User.lookupAll(username=username) if len(users) == 0: user = User(username=username) elif len(users) == 1: user = users[0] elif len(users) > 1: raise Exception("multiple users found with username={}".format(username)) else: raise Exception("This should never happen: len(users)={}".format(len(users))) user.login() login_user(UserWrapper.makeFromUser(user)) def login(self): if current_user.is_authenticated: return redirect('/') if self.auth_plugin is None: self.login_user('anonymous') return redirect('/') form = LoginForm() if form.validate_on_submit(): username = form.username.data password = form.password.data if not self.auth_plugin.authenticate(username, password): flash( 'Invalid username or password or not a member of an ' 'authorized group. Please try again.', 'danger') return render_template( 'login.html', form=form, title=self.company_name, authorized_groups_text=self.authorized_groups_text ) self.login_user(username) return redirect('/') if form.errors: flash(form.errors, 'danger') return render_template( 'login.html', form=form, title=self.company_name, authorized_groups_text=self.authorized_groups_text ) def logout(self): logout_user() return redirect('/') @login_required def sendPage(self, path=None): return self.sendContent("page.html") def mainDisplay(self): def serviceCountSetter(service, ct): def f(): service.target_count = ct return f serviceCounts = list(range(5)) + list(range(10,100,10)) + list(range(100,400,25)) + list(range(400,1001,100)) buttons = Sequence([ Padding(), Button( Sequence([Octicon('shield').color('green'), Span('Lock ALL')]), lambda: [s.lock() for s in service_schema.Service.lookupAll()]), Button( Sequence([Octicon('shield').color('orange'), Span('Prepare ALL')]), lambda: [s.prepare() for s in service_schema.Service.lookupAll()]), Button( Sequence([Octicon('stop').color('red'), Span('Unlock ALL')]), lambda: [s.unlock() for s in service_schema.Service.lookupAll()]), ]) tabs = Tabs( Services=Table( colFun=lambda: [ 'Service', 'Codebase Status', 'Codebase', 'Module', 'Class', 'Placement', 'Active', 'TargetCount', 'Cores', 'RAM', 'Boot Status'], rowFun=lambda: sorted(service_schema.Service.lookupAll(), key=lambda s:s.name), headerFun=lambda x: x, rendererFun=lambda s, field: Subscribed(lambda: Clickable(s.name, "/services/" + s.name) if field == 'Service' else ( Clickable(Sequence([Octicon('stop').color('red'), Span('Unlocked')]), lambda: s.lock()) if s.isUnlocked else Clickable(Sequence([Octicon('shield').color('green'), Span('Locked')]), lambda: s.prepare()) if s.isLocked else Clickable(Sequence([Octicon('shield').color('orange'), Span('Prepared')]), lambda: s.unlock())) if field == 'Codebase Status' else (str(s.codebase) if s.codebase else "") if field == 'Codebase' else s.service_module_name if field == 'Module' else s.service_class_name if field == 'Class' else s.placement if field == 'Placement' else Subscribed(lambda: len(service_schema.ServiceInstance.lookupAll(service=s))) if field == 'Active' else Dropdown(s.target_count, [(str(ct), serviceCountSetter(s, ct)) for ct in serviceCounts]) if field == 'TargetCount' else str(s.coresUsed) if field == 'Cores' else str(s.gbRamUsed) if field == 'RAM' else (Popover(Octicon("alert"), "Failed", Traceback(s.lastFailureReason or "<Unknown>")) if s.isThrottled() else "") if field == 'Boot Status' else "" ), maxRowsPerPage=50 ), Hosts=Table( colFun=lambda: ['Connection', 'IsMaster', 'Hostname', 'RAM ALLOCATION', 'CORE ALLOCATION', 'SERVICE COUNT', 'CPU USE', 'RAM USE'], rowFun=lambda: sorted(service_schema.ServiceHost.lookupAll(), key=lambda s:s.hostname), headerFun=lambda x: x, rendererFun=lambda s,field: Subscribed(lambda: s.connection._identity if field == "Connection" else str(s.isMaster) if field == "IsMaster" else s.hostname if field == "Hostname" else "%.1f / %.1f" % (s.gbRamUsed, s.maxGbRam) if field == "RAM ALLOCATION" else "%s / %s" % (s.coresUsed, s.maxCores) if field == "CORE ALLOCATION" else str(len(service_schema.ServiceInstance.lookupAll(host=s))) if field == "SERVICE COUNT" else "%2.1f" % (s.cpuUse * 100) + "%" if field == "CPU USE" else ("%2.1f" % s.actualMemoryUseGB) + " GB" if field == "RAM USE" else "" ), maxRowsPerPage=50 ) ) return Sequence([buttons, tabs]) def pathToDisplay(self, path, queryArgs): if len(path) and path[0] == 'services': if len(path) == 1: return self.mainDisplay() serviceObj = service_schema.Service.lookupAny(name=path[1]) if serviceObj is None: return Traceback("Unknown service %s" % path[1]) serviceType = serviceObj.instantiateServiceType() if len(path) == 2: return ( Subscribed(lambda: serviceType.serviceDisplay(serviceObj, queryArgs=queryArgs)) .withSerializationContext(serviceObj.getSerializationContext()) ) typename = path[2] schemas = serviceObj.findModuleSchemas() typeObj = None for s in schemas: typeObj = s.lookupFullyQualifiedTypeByName(typename) if typeObj: break if typeObj is None: return Traceback("Can't find fully-qualified type %s" % typename) if len(path) == 3: return ( serviceType.serviceDisplay(serviceObj, objType=typename, queryArgs=queryArgs) .withSerializationContext(serviceObj.getSerializationContext()) ) instance = typeObj.fromIdentity(path[3]) return ( serviceType.serviceDisplay(serviceObj, instance=instance, queryArgs=queryArgs) .withSerializationContext(serviceObj.getSerializationContext()) ) return Traceback("Invalid url path: %s" % path) def addMainBar(self, display): with self.db.view(): current_username = current_user.username return ( HeaderBar( [Subscribed(lambda: Dropdown( "Service", [("All", "/services")] + [(s.name, "/services/" + s.name) for s in sorted(service_schema.Service.lookupAll(), key=lambda s:s.name)] ), ), Dropdown( Octicon("three-bars"), [ (Sequence([Octicon('person'), Span('Logged in as: {}'.format(current_username))]), lambda: None), (Sequence([Octicon('organization'), Span('Authorized Groups: {}'.format(self.authorized_groups_text))]), lambda: None), (Sequence([Octicon('sign-out'), Span('Logout')]), '/logout') ]) ]) + Main(display) ) @login_required def mainSocket(self, ws, path): path = str(path).split("/") queryArgs = dict(request.args) self._logger.info("path = %s", path) reader = None try: self._logger.info("Starting main websocket handler with %s", ws) cells = Cells(self.db) cells.root.setRootSerializationContext(self.db.serializationContext) cells.root.setChild(self.addMainBar(Subscribed(lambda: self.pathToDisplay(path, queryArgs)))) timestamps = [] lastDumpTimestamp = time.time() lastDumpMessages = 0 lastDumpFrames = 0 lastDumpTimeSpentCalculating = 0.0 def readThread(): while not ws.closed: msg = ws.receive() if msg is None: return else: try: jsonMsg = json.loads(msg) cell_id = jsonMsg.get('target_cell') cell = cells[cell_id] if cell is not None: cell.onMessage(jsonMsg) except Exception: self._logger.error("Exception in inbound message: %s", traceback.format_exc()) cells.triggerIfHasDirty() reader = Greenlet.spawn(readThread) while not ws.closed: t0 = time.time() messages = cells.renderMessages() user = self.load_user(current_user.username) if not user.is_authenticated: ws.close() return lastDumpTimeSpentCalculating += time.time() - t0 for message in messages: gevent.socket.wait_write(ws.stream.handler.socket.fileno()) ws.send(json.dumps(message)) lastDumpMessages += 1 lastDumpFrames += 1 if time.time() - lastDumpTimestamp > 5.0: self._logger.info("In the last %.2f seconds, spent %.2f seconds calculating %s messages over %s frames", time.time() - lastDumpTimestamp, lastDumpTimeSpentCalculating, lastDumpMessages, lastDumpFrames ) lastDumpFrames = 0 lastDumpMessages = 0 lastDumpTimeSpentCalculating = 0 lastDumpTimestamp = time.time() ws.send(json.dumps("postscripts")) cells.wait() timestamps.append(time.time()) if len(timestamps) > MAX_FPS: timestamps = timestamps[-MAX_FPS+1:] if (time.time() - timestamps[0]) < 1.0: sleep(1.0 / MAX_FPS + .001) except Exception: self._logger.error("Websocket handler error: %s", traceback.format_exc()) finally: if reader: reader.join() @login_required def echoSocket(self, ws): while not ws.closed: message = ws.receive() if message is not None: ws.send(message) @login_required def sendContent(self, path): own_dir = os.path.dirname(__file__) return send_from_directory(os.path.join(own_dir, "content"), path) @staticmethod def serviceDisplay(serviceObject, instance=None, objType=None, queryArgs=None): c = Configuration.lookupAny(service=serviceObject) return Card(Text("Host: " + c.hostname) + Text("Port: " + str(c.port)))