async def put_config_validation(self, request: Request) -> StreamResponse: config_id = request.match_info["config_id"] js = await self.json_from_request(request) js["id"] = config_id config_model = from_js(js, ConfigValidation) model = await self.config_handler.put_config_validation(config_model) return await single_result(request, to_js(model))
def test_parse_broken(config_json: Json) -> None: # config_json is a valid parsable config cfg = deepcopy(config_json) # adjust the config: rename web_hosts -> hosts, and web_port -> port hosts = cfg["resotocore"]["api"]["web_hosts"] port = cfg["resotocore"]["api"]["web_port"] cfg["resotocore"]["api"]["hosts"] = hosts cfg["resotocore"]["api"]["port"] = port del cfg["resotocore"]["api"]["web_hosts"] del cfg["resotocore"]["api"]["web_port"] # parse this configuration parsed = parse_config(parse_args(["--analytics-opt-out"]), cfg) parsed_json = to_js(parsed.editable, strip_attr="kind") # web_hosts and web_port were not available and are reverted to the default values default = EditableConfig() assert parsed.api.web_hosts != hosts assert parsed.api.web_hosts == default.api.web_hosts assert parsed.api.web_port != port assert parsed.api.web_port == default.api.web_port # other config values are still unchanged assert parsed_json["cli"] == config_json["resotocore"]["cli"] assert parsed_json["runtime"] == config_json["resotocore"]["runtime"] assert parsed_json["graph_update"] == config_json["resotocore"][ "graph_update"]
def to_json(o: Job, **_: object) -> Json: wait = { "wait_trigger": to_js(o.wait[0]), "wait_timeout": to_json(o.wait[1]) } if o.wait else {} env = {"environment": o.environment} if o.environment else {} return { "id": o.id, "name": o.name, "command": to_js(o.command), "trigger": to_js(o.trigger), "timeout": to_json(o.timeout), "active": o.active, **env, **wait, }
async def update_configs_model(self, update: List[Kind]) -> Model: async with self.session.patch(self.base_path + "/configs/model", json=to_js(update)) as response: model_json = await response.json() model = Model.from_kinds([ from_js(kind, Kind) for kind in model_json["kinds"].values() ]) # type: ignore return model
def cmd_json(cmd: CLICommand) -> Json: return { "name": cmd.name, "info": cmd.info(), "help": cmd.help(), "args": to_js(cmd.args_info(), force_dict=True), "source": cmd.allowed_in_source_position, }
async def merge_graph(self, request: Request) -> StreamResponse: log.info("Received merge_graph request") graph_id = request.match_info.get("graph_id", "resoto") db = self.db.get_graph_db(graph_id) it = self.to_line_generator(request) info = await merge_graph_process(db, self.event_sender, self.args, it, self.merge_max_wait_time, None) return web.json_response(to_js(info))
async def update_subscriber( self, uid: str, subscriptions: List[Subscription]) -> Optional[Subscriber]: async with self.session.put(self.base_path + f"/subscriber/{uid}", json=to_js(subscriptions)) as r: if r.status == 200: return from_js(await r.json(), Subscriber) else: raise AttributeError(await r.text())
def insert_system_data() -> SystemData: system = SystemData(uuid_str(), utc(), 1) log.info(f"Create new system data entry: {system}") db.insert_document("system_data", { "_key": "system", **to_js(system) }, overwrite=True) return system
async def add_subscription(self, request: Request) -> StreamResponse: subscriber_id = request.match_info["subscriber_id"] event_type = request.match_info["event_type"] timeout = timedelta(seconds=int(request.query.get("timeout", "60"))) wait_for_completion = request.query.get("wait_for_completion", "true").lower() != "false" sub = await self.subscription_handler.add_subscription( subscriber_id, event_type, wait_for_completion, timeout) return await single_result(request, to_js(sub))
def test_json_marshalling_works() -> None: m = ModelFoo(1, "some foo", 23) js = to_js(m) js["identity"] = 1 js["a"] = "some foo" js["b"] = 23 again = from_js(js, ModelFoo) d = DeepDiff(m, again, truncate_datetime="second") assert len(d) == 0
async def test_merge_process(event_sender: AnalyticsEventSender, graph_db: ArangoGraphDB, foo_kinds: List[Kind]) -> None: # set explicitly (is done in main explicitly as well) set_start_method("spawn") # wipe any existing data await graph_db.wipe() # store the model in db, so it can be loaded by the sub process graph_db.db.collection("model").insert_many([to_js(a) for a in foo_kinds]) # define args to parse for the sub process config = empty_config([ "--graphdb-username", "test", "--graphdb-password", "test", "--graphdb-database", "test" ]) # create sample graph data to insert graph = create_graph("test") await outer_edge_db(graph_db.db, "deferred_outer_edges").create_update_schema() async def iterator() -> AsyncGenerator[bytes, None]: for node in graph.nodes(): yield bytes(json.dumps(graph.nodes[node]), "utf-8") for from_node, to_node, data in graph.edges(data=True): yield bytes( json.dumps({ "from": from_node, "to": to_node, "edge_type": data["edge_type"] }), "utf-8") yield bytes( json.dumps({ "from_selector": { "node_id": "id_123" }, "to_selector": { "node_id": "id_456" }, "edge_type": "delete" }), "utf-8", ) result = await merge_graph_process(graph_db, event_sender, config, iterator(), timedelta(seconds=30), None, TaskId("test_task_123")) assert result == GraphUpdate(112, 1, 0, 212, 0, 0) elem = graph_db.db.collection("deferred_outer_edges").all().next() assert elem["_key"] == "test_task_123" assert elem["task_id"] == "test_task_123" assert elem["edges"][0] == { "from_node": "id_123", "to_node": "id_456", "edge_type": "delete" }
def to_json(o: Workflow, **_: object) -> Json: env = {"environment": o.environment} if o.environment else {} return { "id": o.id, "name": o.name, "steps": to_json(o.steps), "triggers": to_json(o.triggers), "on_surpass": to_js(o.on_surpass), **env, }
async def update_state(self, wi: RunningTask, message: Optional[Message]) -> None: bind = { "id": f"{self.collection_name}/{wi.id}", "current_state_name": wi.current_state.name, "current_state_snapshot": wi.current_state.export_state(), } if message: bind["message"] = to_js(message) aql = self.__update_state_with_message() else: aql = self.__update_state() await self.db.aql(aql, bind_vars=bind)
async def test_workflows_command(cli: CLI, task_handler: TaskHandlerService, test_workflow: Workflow) -> None: async def execute(cmd: str) -> List[JsonElement]: ctx = CLIContext(cli.cli_env) return (await cli.execute_cli_command(cmd, stream.list, ctx))[0] # type: ignore assert await execute("workflows list") == ["test_workflow"] assert await execute("workflows show test_workflow") == [ to_js(test_workflow) ] wf = await execute("workflows run test_workflow") assert wf[0].startswith( "Workflow test_workflow started with id") # type: ignore assert len(await execute("workflows running")) == 1
async def core_client( client_session: ClientSession, foo_kinds: List[Kind], test_db: StandardDatabase ) -> AsyncIterator[ApiClient]: """ Note: adding this fixture to a test: a complete resotocore process is started. The fixture ensures that the underlying process has entered the ready state. It also ensures to clean up the process, when the test is done. """ # wipe and cleanly import the test model test_db.collection("model").truncate() test_db.collection("model").insert_many([{"_key": elem.fqn, **to_js(elem)} for elem in foo_kinds]) process = Process( target=run, args=( [ "--graphdb-database", "test", "--graphdb-username", "test", "--graphdb-password", "test", "--debug", "--no-tls", ], ), ) process.start() ready = False count = 10 while not ready: await sleep(0.5) with suppress(Exception): async with client_session.get("http://localhost:8900/system/ready"): ready = True count -= 1 if count == 0: raise AssertionError("Process does not came up as expected") yield ApiClient("http://localhost:8900", None) # terminate the process process.terminate() process.join(5) # if it is still running, kill it if process.is_alive(): process.kill() process.join() process.close()
async def test_merge_process( event_sender: AnalyticsEventSender, graph_db: ArangoGraphDB, foo_kinds: List[Kind] ) -> None: # set explicitly (is done in main explicitly as well) set_start_method("spawn") # wipe any existing data await graph_db.wipe() # store the model in db, so it can be loaded by the sub process graph_db.db.collection("model").insert_many([to_js(a) for a in foo_kinds]) # define args to parse for the sub process args = parse_args(["--graphdb-username", "test", "--graphdb-password", "test", "--graphdb-database", "test"]) # create sample graph data to insert graph = create_graph("test") async def iterator() -> AsyncGenerator[bytes, None]: for node in graph.nodes(): yield bytes(json.dumps(graph.nodes[node]), "utf-8") for from_node, to_node, data in graph.edges(data=True): yield bytes(json.dumps({"from": from_node, "to": to_node, "edge_type": data["edge_type"]}), "utf-8") result = await merge_graph_process(graph_db, event_sender, args, iterator(), timedelta(seconds=30), None) assert result == GraphUpdate(112, 1, 0, 212, 0, 0)
def to_doc(self, elem: T) -> Json: js = to_js(elem) js["_key"] = self.key_of(elem) return js
def test_config_entity_roundtrip() -> None: entity = ConfigEntity(ConfigId("test"), {"test": 1}, "test") again = from_js(to_js(entity), ConfigEntity) assert entity == again
async def explain(self, request: Request) -> StreamResponse: graph_db, query_model = await self.graph_query_model_from_request( request) result = await graph_db.explain(query_model) return web.json_response(to_js(result))
async def update_model(self, request: Request) -> StreamResponse: js = await self.json_from_request(request) kinds: List[Kind] = from_js(js, List[Kind]) model = await self.model_handler.update_model(kinds) return await single_result(request, to_js(model))
async def get_model(self, request: Request) -> StreamResponse: md = await self.model_handler.load_model() return await single_result(request, to_js(md))
async def delete_subscription(self, request: Request) -> StreamResponse: subscriber_id = request.match_info["subscriber_id"] event_type = request.match_info["event_type"] sub = await self.subscription_handler.remove_subscription( subscriber_id, event_type) return await single_result(request, to_js(sub))
class Api: def __init__( self, db: DbAccess, model_handler: ModelHandler, subscription_handler: SubscriptionHandler, workflow_handler: TaskHandlerService, message_bus: MessageBus, event_sender: AnalyticsEventSender, worker_task_queue: WorkerTaskQueue, cert_handler: CertificateHandler, config_handler: ConfigHandler, cli: CLI, query_parser: QueryParser, config: CoreConfig, ): self.db = db self.model_handler = model_handler self.subscription_handler = subscription_handler self.workflow_handler = workflow_handler self.message_bus = message_bus self.event_sender = event_sender self.worker_task_queue = worker_task_queue self.cert_handler = cert_handler self.config_handler = config_handler self.cli = cli self.query_parser = query_parser self.config = config self.app = web.Application( # note on order: the middleware is passed in the order provided. middlewares=[ metrics_handler, auth_handler(config.args.psk, AlwaysAllowed), cors_handler, error_handler(config, event_sender), default_middleware(self), ]) self.app.on_response_prepare.append(on_response_prepare) self.session: Optional[ClientSession] = None self.in_shutdown = False self.websocket_handler: Dict[str, Tuple[Future[Any], WebSocketResponse]] = {} path_part = config.api.web_path.strip().strip("/").strip() web_path = "" if path_part == "" else f"/{path_part}" self.__add_routes(web_path) def __add_routes(self, prefix: str) -> None: static_path = os.path.abspath(os.path.dirname(__file__) + "/../static") ui_route: List[AbstractRouteDef] = ( [web.static(f"{prefix}/ui/", self.config.api.ui_path)] if self.config.api.ui_path and Path(self.config.api.ui_path).exists() else [web.get(f"{prefix}/ui/index.html", self.no_ui)]) tsdb_route = ([ web.route(METH_ANY, prefix + "/tsdb/{tail:.+}", tsdb(self)) ] if self.config.api.tsdb_proxy_url else []) self.app.add_routes([ # Model operations web.get(prefix + "/model", self.get_model), web.get(prefix + "/model/uml", self.model_uml), web.patch(prefix + "/model", self.update_model), # CRUD Graph operations web.get(prefix + "/graph", self.list_graphs), web.get(prefix + "/graph/{graph_id}", self.get_node), web.post(prefix + "/graph/{graph_id}", self.create_graph), web.delete(prefix + "/graph/{graph_id}", self.wipe), # search the graph web.post(prefix + "/graph/{graph_id}/search/raw", self.raw), web.post(prefix + "/graph/{graph_id}/search/explain", self.explain), web.post(prefix + "/graph/{graph_id}/search/list", self.query_list), web.post(prefix + "/graph/{graph_id}/search/graph", self.query_graph_stream), web.post(prefix + "/graph/{graph_id}/search/aggregate", self.query_aggregation), # maintain the graph web.patch(prefix + "/graph/{graph_id}/nodes", self.update_nodes), web.post(prefix + "/graph/{graph_id}/merge", self.merge_graph), web.post(prefix + "/graph/{graph_id}/batch/merge", self.update_merge_graph_batch), web.get(prefix + "/graph/{graph_id}/batch", self.list_batches), web.post(prefix + "/graph/{graph_id}/batch/{batch_id}", self.commit_batch), web.delete(prefix + "/graph/{graph_id}/batch/{batch_id}", self.abort_batch), # node specific actions web.post( prefix + "/graph/{graph_id}/node/{node_id}/under/{parent_node_id}", self.create_node), web.get(prefix + "/graph/{graph_id}/node/{node_id}", self.get_node), web.patch(prefix + "/graph/{graph_id}/node/{node_id}", self.update_node), web.delete(prefix + "/graph/{graph_id}/node/{node_id}", self.delete_node), web.patch( prefix + "/graph/{graph_id}/node/{node_id}/section/{section}", self.update_node), # Subscriptions web.get(prefix + "/subscribers", self.list_all_subscriptions), web.get(prefix + "/subscribers/for/{event_type}", self.list_subscription_for_event), # Subscription web.get(prefix + "/subscriber/{subscriber_id}", self.get_subscriber), web.put(prefix + "/subscriber/{subscriber_id}", self.update_subscriber), web.delete(prefix + "/subscriber/{subscriber_id}", self.delete_subscriber), web.post(prefix + "/subscriber/{subscriber_id}/{event_type}", self.add_subscription), web.delete(prefix + "/subscriber/{subscriber_id}/{event_type}", self.delete_subscription), web.get(prefix + "/subscriber/{subscriber_id}/handle", self.handle_subscribed), # CLI web.post(prefix + "/cli/evaluate", self.evaluate), web.post(prefix + "/cli/execute", self.execute), web.get(prefix + "/cli/info", self.cli_info), # Event operations web.get(prefix + "/events", self.handle_events), # Worker operations web.get(prefix + "/work/queue", self.handle_work_tasks), web.get(prefix + "/work/create", self.create_work), web.get(prefix + "/work/list", self.list_work), # Serve static filed web.get(prefix, self.redirect_to_api_doc), web.static(prefix + "/static", static_path), # metrics web.get(prefix + "/metrics", self.metrics), # config operations web.get(prefix + "/configs", self.list_configs), web.put(prefix + "/config/{config_id}", self.put_config), web.get(prefix + "/config/{config_id}", self.get_config), web.patch(prefix + "/config/{config_id}", self.patch_config), web.delete(prefix + "/config/{config_id}", self.delete_config), # config model operations web.get(prefix + "/configs/validation", self.list_config_models), web.get(prefix + "/configs/model", self.get_configs_model), web.patch(prefix + "/configs/model", self.update_configs_model), web.put(prefix + "/config/{config_id}/validation", self.put_config_validation), web.get(prefix + "/config/{config_id}/validation", self.get_config_validation), # ca operations web.get(prefix + "/ca/cert", self.certificate), web.post(prefix + "/ca/sign", self.sign_certificate), # system operations web.get(prefix + "/system/ping", self.ping), web.get(prefix + "/system/ready", self.ready), # forwards web.get(prefix + "/tsdb", self.forward("/tsdb/")), web.get(prefix + "/ui", self.forward("/ui/index.html")), web.get(prefix + "/ui/", self.forward("/ui/index.html")), *ui_route, *tsdb_route, ]) SwaggerFile( self.app, spec_file=f"{static_path}/api-doc.yaml", swagger_ui_settings=SwaggerUiSettings(path=prefix + "/api-doc", layout="BaseLayout", docExpansion="none"), ) async def start(self) -> None: pass async def stop(self) -> None: if not self.in_shutdown: self.in_shutdown = True for ws_id in list(self.websocket_handler): await clean_ws_handler(ws_id, self.websocket_handler) if self.session: await self.session.close() @staticmethod def forward(to: str) -> Callable[[Request], Awaitable[StreamResponse]]: async def forward_to(_: Request) -> StreamResponse: return web.HTTPFound(to) return forward_to @staticmethod async def ping(_: Request) -> StreamResponse: return web.HTTPOk(text="pong", content_type="text/plain") @staticmethod async def ready(_: Request) -> StreamResponse: return web.HTTPOk(text="ok") async def list_configs(self, request: Request) -> StreamResponse: return await self.stream_response_from_gen( request, self.config_handler.list_config_ids()) async def get_config(self, request: Request) -> StreamResponse: config_id = ConfigId(request.match_info["config_id"]) accept = request.headers.get("accept", "application/json") not_found = HTTPNotFound(text="No config with this id") if accept == "application/yaml": yml = await self.config_handler.config_yaml(config_id) return web.Response( body=yml.encode("utf-8"), content_type="application/yaml") if yml else not_found else: config = await self.config_handler.get_config(config_id) if config: headers = {"Resoto-Config-Revision": config.revision} return await single_result(request, config.config, headers) else: return not_found async def put_config(self, request: Request) -> StreamResponse: config_id = ConfigId(request.match_info["config_id"]) validate = request.query.get("validate", "true").lower() != "false" config = await self.json_from_request(request) result = await self.config_handler.put_config( ConfigEntity(config_id, config), validate) headers = {"Resoto-Config-Revision": result.revision} return await single_result(request, result.config, headers) async def patch_config(self, request: Request) -> StreamResponse: config_id = ConfigId(request.match_info["config_id"]) patch = await self.json_from_request(request) updated = await self.config_handler.patch_config( ConfigEntity(config_id, patch)) headers = {"Resoto-Config-Revision": updated.revision} return await single_result(request, updated.config, headers) async def delete_config(self, request: Request) -> StreamResponse: config_id = ConfigId(request.match_info["config_id"]) await self.config_handler.delete_config(config_id) return HTTPNoContent() async def list_config_models(self, request: Request) -> StreamResponse: return await self.stream_response_from_gen( request, self.config_handler.list_config_validation_ids()) async def get_config_validation(self, request: Request) -> StreamResponse: config_id = request.match_info["config_id"] model = await self.config_handler.get_config_validation(config_id) return await single_result(request, to_js(model)) if model else HTTPNotFound( text="No model for this config.") async def get_configs_model(self, request: Request) -> StreamResponse: model = await self.config_handler.get_configs_model() return await single_result(request, to_js(model)) async def update_configs_model(self, request: Request) -> StreamResponse: js = await self.json_from_request(request) kinds: List[Kind] = from_js(js, List[Kind]) model = await self.config_handler.update_configs_model(kinds) return await single_result(request, to_js(model)) async def put_config_validation(self, request: Request) -> StreamResponse: config_id = request.match_info["config_id"] js = await self.json_from_request(request) js["id"] = config_id config_model = from_js(js, ConfigValidation) model = await self.config_handler.put_config_validation(config_model) return await single_result(request, to_js(model)) async def certificate(self, _: Request) -> StreamResponse: cert, fingerprint = self.cert_handler.authority_certificate headers = { "SHA256-Fingerprint": fingerprint, "Content-Disposition": 'attachment; filename="resoto_root_ca.pem"', } if self.config.args.psk: headers["Authorization"] = "Bearer " + encode_jwt( {"sha256_fingerprint": fingerprint}, self.config.args.psk) return HTTPOk(headers=headers, body=cert, content_type="application/x-pem-file") async def sign_certificate(self, request: Request) -> StreamResponse: csr_bytes = await request.content.read() cert, fingerprint = self.cert_handler.sign(csr_bytes) headers = {"SHA256-Fingerprint": fingerprint} return HTTPOk(headers=headers, body=cert, content_type="application/x-pem-file") @staticmethod async def metrics(_: Request) -> StreamResponse: resp = web.Response(body=prometheus_client.generate_latest()) resp.content_type = prometheus_client.CONTENT_TYPE_LATEST return resp async def list_all_subscriptions(self, request: Request) -> StreamResponse: subscribers = await self.subscription_handler.all_subscribers() return await single_result(request, to_json(subscribers)) async def get_subscriber(self, request: Request) -> StreamResponse: subscriber_id = SubscriberId(request.match_info["subscriber_id"]) subscriber = await self.subscription_handler.get_subscriber( subscriber_id) return self.optional_json(subscriber, f"No subscriber with id {subscriber_id}") async def list_subscription_for_event(self, request: Request) -> StreamResponse: event_type = request.match_info["event_type"] subscribers = await self.subscription_handler.list_subscriber_for( event_type) return await single_result(request, to_json(subscribers)) async def update_subscriber(self, request: Request) -> StreamResponse: subscriber_id = SubscriberId(request.match_info["subscriber_id"]) body = await self.json_from_request(request) subscriptions = from_js(body, List[Subscription]) sub = await self.subscription_handler.update_subscriptions( subscriber_id, subscriptions) return await single_result(request, to_json(sub)) async def delete_subscriber(self, request: Request) -> StreamResponse: subscriber_id = SubscriberId(request.match_info["subscriber_id"]) await self.subscription_handler.remove_subscriber(subscriber_id) return web.HTTPNoContent() async def add_subscription(self, request: Request) -> StreamResponse: subscriber_id = SubscriberId(request.match_info["subscriber_id"]) event_type = request.match_info["event_type"] timeout = timedelta(seconds=int(request.query.get("timeout", "60"))) wait_for_completion = request.query.get("wait_for_completion", "true").lower() != "false" sub = await self.subscription_handler.add_subscription( subscriber_id, event_type, wait_for_completion, timeout) return await single_result(request, to_js(sub)) async def delete_subscription(self, request: Request) -> StreamResponse: subscriber_id = SubscriberId(request.match_info["subscriber_id"]) event_type = request.match_info["event_type"] sub = await self.subscription_handler.remove_subscription( subscriber_id, event_type) return await single_result(request, to_js(sub)) async def handle_subscribed(self, request: Request) -> StreamResponse: subscriber_id = SubscriberId(request.match_info["subscriber_id"]) subscriber = await self.subscription_handler.get_subscriber( subscriber_id) if subscriber_id in self.message_bus.active_listener: log.info( f"There is already a listener for subscriber: {subscriber_id}. Reject." ) return web.HTTPTooManyRequests( text="Only one connection per subscriber is allowed!") elif subscriber and subscriber.subscriptions: pending = await self.workflow_handler.list_all_pending_actions_for( subscriber) return await self.listen_to_events( request, subscriber_id, list(subscriber.subscriptions.keys()), pending) else: return web.HTTPNotFound( text= f"No subscriber with this id: {subscriber_id} or no subscriptions" ) async def redirect_to_api_doc(self, request: Request) -> StreamResponse: raise web.HTTPFound("api-doc") async def handle_events(self, request: Request) -> StreamResponse: show = request.query["show"].split( ",") if "show" in request.query else ["*"] return await self.listen_to_events(request, SubscriberId(str(uuid.uuid1())), show) async def listen_to_events( self, request: Request, listener_id: SubscriberId, event_types: List[str], initial_messages: Optional[Sequence[Message]] = None, ) -> WebSocketResponse: async def handle_message(msg: str) -> None: js = json.loads(msg) if "data" in js: js["data"]["subscriber_id"] = listener_id message: Message = from_js(js, Message) if isinstance(message, Action): raise AttributeError( "Actors should not emit action messages. ") elif isinstance(message, ActionDone): await self.workflow_handler.handle_action_done(message) elif isinstance(message, ActionError): await self.workflow_handler.handle_action_error(message) else: await self.message_bus.emit(message) return await accept_websocket( request, handle_incoming=handle_message, outgoing_context=partial(self.message_bus.subscribe, listener_id, event_types), websocket_handler=self.websocket_handler, initial_messages=initial_messages, ) async def handle_work_tasks(self, request: Request) -> WebSocketResponse: worker_id = WorkerId(uuid_str()) task_param = request.query.get("task") if not task_param: raise AttributeError( "A worker needs to define at least one task that it can perform" ) attrs = { k: re.split("\\s*,\\s*", v) for k, v in request.query.items() if k != "task" } task_descriptions = [ WorkerTaskDescription(name, attrs) for name in re.split("\\s*,\\s*", task_param) ] async def handle_message(msg: str) -> None: tr = from_js(json.loads(msg), WorkerTaskResult) if tr.result == "error": error = tr.error if tr.error else "worker signalled error without detailed error message" await self.worker_task_queue.error_task( worker_id, tr.task_id, error) elif tr.result == "done": await self.worker_task_queue.acknowledge_task( worker_id, tr.task_id, tr.data) else: log.info(f"Do not understand this message: {msg}") def task_json(task: WorkerTask) -> str: return to_js_str(task.to_json()) return await accept_websocket( request, handle_incoming=handle_message, outgoing_context=partial(self.worker_task_queue.attach, worker_id, task_descriptions), websocket_handler=self.websocket_handler, outgoing_fn=task_json, ) async def create_work(self, request: Request) -> StreamResponse: attrs = {k: v for k, v in request.query.items() if k != "task"} future = asyncio.get_event_loop().create_future() task = WorkerTask(TaskId(uuid_str()), "test", attrs, { "some": "data", "foo": "bla" }, future, timedelta(seconds=3)) await self.worker_task_queue.add_task(task) await future return web.HTTPOk() async def list_work(self, request: Request) -> StreamResponse: def wt_to_js(ip: WorkerTaskInProgress) -> Json: return { "task": ip.task.to_json(), "worker": ip.worker.worker_id, "retry_counter": ip.retry_counter, "deadline": to_json(ip.deadline), } return web.json_response([ wt_to_js(ot) for ot in self.worker_task_queue.outstanding_tasks.values() ]) async def model_uml(self, request: Request) -> StreamResponse: output = request.query.get("output", "svg") show = request.query["show"].split( ",") if "show" in request.query else None hide = request.query["hide"].split( ",") if "hide" in request.query else None with_inheritance = request.query.get("with_inheritance", "true") != "false" with_base_classes = request.query.get("with_base_classes", "true") != "false" with_subclasses = request.query.get("with_subclasses", "false") != "false" dependency = set(request.query["dependency"].split( ",")) if "dependency" in request.query else None with_predecessors = request.query.get("with_predecessors", "false") != "false" with_successors = request.query.get("with_successors", "false") != "false" with_properties = request.query.get("with_properties", "true") != "false" link_classes = request.query.get("link_classes", "false") != "false" result = await self.model_handler.uml_image( output=output, show_packages=show, hide_packages=hide, with_inheritance=with_inheritance, with_base_classes=with_base_classes, with_subclasses=with_subclasses, dependency_edges=dependency, with_predecessors=with_predecessors, with_successors=with_successors, with_properties=with_properties, link_classes=link_classes, ) response = web.StreamResponse() mt = {"svg": "image/svg+xml", "png": "image/png"} response.headers["Content-Type"] = mt[output] await response.prepare(request) await response.write_eof(result) return response async def get_model(self, request: Request) -> StreamResponse: md = await self.model_handler.load_model() return await single_result(request, to_js(md.kinds.values())) async def update_model(self, request: Request) -> StreamResponse: js = await self.json_from_request(request) kinds: List[Kind] = from_js(js, List[Kind]) model = await self.model_handler.update_model(kinds) return await single_result(request, to_js(model)) async def get_node(self, request: Request) -> StreamResponse: graph_id = request.match_info.get("graph_id", "resoto") node_id = request.match_info.get("node_id", "root") graph = self.db.get_graph_db(graph_id) model = await self.model_handler.load_model() node = await graph.get_node(model, node_id) if node is None: return web.HTTPNotFound( text=f"No such node with id {node_id} in graph {graph_id}") else: return await single_result(request, node) async def create_node(self, request: Request) -> StreamResponse: graph_id = request.match_info.get("graph_id", "resoto") node_id = request.match_info.get("node_id", "some_existing") parent_node_id = request.match_info.get("parent_node_id", "root") graph = self.db.get_graph_db(graph_id) item = await self.json_from_request(request) md = await self.model_handler.load_model() node = await graph.create_node(md, node_id, item, parent_node_id) return await single_result(request, node) async def update_node(self, request: Request) -> StreamResponse: graph_id = request.match_info.get("graph_id", "resoto") node_id = request.match_info.get("node_id", "some_existing") section = section_of(request) graph = self.db.get_graph_db(graph_id) patch = await self.json_from_request(request) md = await self.model_handler.load_model() node = await graph.update_node(md, node_id, patch, False, section) return await single_result(request, node) async def delete_node(self, request: Request) -> StreamResponse: graph_id = request.match_info.get("graph_id", "resoto") node_id = request.match_info.get("node_id", "some_existing") if node_id == "root": raise AttributeError("Root node can not be deleted!") graph = self.db.get_graph_db(graph_id) await graph.delete_node(node_id) return web.HTTPNoContent() async def update_nodes(self, request: Request) -> StreamResponse: graph_id = request.match_info.get("graph_id", "resoto") allowed = {*Section.content, "id", "revision"} updates: Dict[str, Json] = {} async for elem in self.to_json_generator(request): keys = set(elem.keys()) assert keys.issubset( allowed), f"Invalid json. Allowed keys are: {allowed}" assert "id" in elem, f"No id given for element {elem}" assert keys.intersection( Section.content), f"No update provided for element {elem}" uid = elem["id"] assert uid not in updates, f"Only one update allowed per id! {elem}" del elem["id"] updates[uid] = elem db = self.db.get_graph_db(graph_id) model = await self.model_handler.load_model() result_gen = db.update_nodes(model, updates) return await self.stream_response_from_gen(request, result_gen) async def list_graphs(self, request: Request) -> StreamResponse: graphs = await self.db.list_graphs() return await single_result(request, graphs) async def create_graph(self, request: Request) -> StreamResponse: graph_id = request.match_info.get("graph_id", "resoto") if "_" in graph_id: raise AttributeError("Graph name should not have underscores!") graph = await self.db.create_graph(graph_id) model = await self.model_handler.load_model() root = await graph.get_node(model, "root") return web.json_response(root) async def merge_graph(self, request: Request) -> StreamResponse: log.info("Received merge_graph request") graph_id = request.match_info.get("graph_id", "resoto") task_id: Optional[TaskId] = None if tid := request.headers.get("Resoto-Worker-Task-Id"): task_id = TaskId(tid) db = self.db.get_graph_db(graph_id) it = self.to_line_generator(request) info = await merge_graph_process( db, self.event_sender, self.config, it, self.config.graph_update.merge_max_wait_time(), None, task_id) return web.json_response(to_js(info))
async def update_many(self, elements: List[T]) -> None: for elem in elements: key = self.key_fn(elem) self.items[key] = to_js(elem)
async def update(self, t: T) -> T: self.items[self.key_fn(t)] = to_js(t) return t
async def put_config_validation(self, cfg: ConfigValidation) -> ConfigValidation: async with self.session.put(self.base_path + f"/config/{cfg.id}/validation", json=to_js(cfg)) as response: return from_js(await response.json(), ConfigValidation)
def roundtrip(obj: Any) -> None: js = to_js(obj) again = from_js(js, type(obj)) assert DeepDiff(obj, again) == {}, f"Json: {js} serialized as {again}"
async def get_config_validation(self, request: Request) -> StreamResponse: config_id = request.match_info["config_id"] model = await self.config_handler.get_config_validation(config_id) return await single_result(request, to_js(model)) if model else HTTPNotFound( text="No model for this config.")
def to_json(obj: BaseResource) -> Json: return {"kind": obj.kind(), **to_js(obj)}
async def get_configs_model(self, request: Request) -> StreamResponse: model = await self.config_handler.get_configs_model() return await single_result(request, to_js(model))