async def test_events(event_graph_db: EventGraphDB, foo_model: Model, event_sender: InMemoryEventSender) -> None: await event_graph_db.create_node(foo_model, "some_other", to_json(Foo("some_other", "foo")), "root") await event_graph_db.update_node(foo_model, "some_other", {"name": "bla"}, False, "reported") await event_graph_db.delete_node("some_other") await event_graph_db.merge_graph(create_graph("yes or no", width=1), foo_model) await event_graph_db.merge_graph(create_graph("maybe", width=1), foo_model, "batch1", True) # make sure all events will arrive await asyncio.sleep(0.1) # ensure the correct count and order of events assert [a.kind for a in event_sender.events] == [ CoreEvent.NodeCreated, CoreEvent.NodeUpdated, CoreEvent.NodeDeleted, CoreEvent.GraphMerged, CoreEvent.BatchUpdateGraphMerged, ] merge_event = AccessJson(event_sender.events[3].context) assert merge_event.graph == event_graph_db.graph_name assert merge_event.providers == ["collector"] assert merge_event.batch is False
async def get_node(self, graph: str, node_id: str) -> AccessJson: async with self.session.get( self.base_path + f"/graph/{graph}/node/{node_id}") as response: if response.status == 200: return AccessJson(await response.json()) else: raise AttributeError(await response.text())
async def create_node(self, graph: str, parent_node_id: str, node_id: str, node: Json) -> AccessJson: async with self.session.post( self.base_path + f"/graph/{graph}/node/{node_id}/under/{parent_node_id}", json=node) as response: if response.status == 200: return AccessJson(await response.json()) else: raise AttributeError(await response.text())
async def patch_node(self, graph: str, node_id: str, node: Json, section: Optional[str] = None) -> AccessJson: section_path = f"/section/{section}" if section else "" async with self.session.patch( self.base_path + f"/graph/{graph}/node/{node_id}{section_path}", json=node) as response: if response.status == 200: return AccessJson(await response.json()) else: raise AttributeError(await response.text())
def test_resolve_graph_data() -> None: g = multi_cloud_graph("account") graph = GraphAccess(g) graph.resolve() # ancestor data should be stored in metadata n1 = AccessJson(graph.node("child_parent_region_account_cloud_gcp_1_europe_1_0")) # type: ignore assert n1.refs.region_id == "region_account_cloud_gcp_1_europe" assert n1.ancestors.account.reported.id == "id_account_cloud_gcp_1" assert n1.ancestors.account.reported.name == "name_account_cloud_gcp_1" assert n1.ancestors.region.reported.id == "id_region_account_cloud_gcp_1_europe" assert n1.ancestors.region.reported.name == "name_region_account_cloud_gcp_1_europe" # make sure there is no summary assert n1.descendant_summary == AccessNone(None) r1 = AccessJson(graph.node("region_account_cloud_gcp_1_europe")) # type: ignore assert r1.metadata.descendant_summary == {"child": 9} assert r1.metadata.descendant_count == 9 r2 = AccessJson(graph.node("account_cloud_gcp_1")) # type: ignore assert r2.metadata.descendant_summary == {"child": 54, "region": 6} assert r2.metadata.descendant_count == 60 r3 = AccessJson(graph.node("cloud_gcp")) # type: ignore assert r3.metadata.descendant_summary == {"child": 162, "region": 18, "account": 3} assert r3.metadata.descendant_count == 183
async def test_query_merge(filled_graph_db: ArangoGraphDB, foo_model: Model) -> None: q = parse_query("is(foo) --> is(bla) { " "foo.bar.parents[]: <-[1:]-, " "foo.child: -->, " "walk: <-- -->, " "bla.agg: aggregate(sum(1) as count): <-[0:]- " "}") async with await filled_graph_db.search_list(QueryModel(q, foo_model), with_count=True) as cursor: assert cursor.count() == 100 async for bla in cursor: b = AccessJson(bla) assert b.reported.kind == "bla" assert len(b.foo.bar.parents) == 4 for parent in b.foo.bar.parents: assert parent.reported.kind in ["foo", "cloud", "graph_root"] assert b.walk.reported.kind == "bla" assert b.foo.child == AccessNone() assert b.bla.agg == [{"count": 5}]
def test_access_json() -> None: js = {"a": "a", "b": {"c": "c", "d": {"e": "e", "f": [0, 1, 2, 3, 4]}}} access = AccessJson(js, "null") assert access.a == "a" assert access.b.d.f[2] == 2 assert str(access.foo.bla.bar[23].now) is "null" assert json.dumps(access.b.d, sort_keys=True) == '{"e": "e", "f": [0, 1, 2, 3, 4]}' assert access["a"] == "a" assert access["b"]["d"]["f"][2] == 2 assert str(access["foo"]["bla"]["bar"][23]["now"]) is "null" assert json.dumps(access["b"]["d"], sort_keys=True) == '{"e": "e", "f": [0, 1, 2, 3, 4]}' assert [a for a in access] == ["a", "b"] assert [a for a in access.doesnt.exist] == [] assert [a for a in access.b.d.items()] == [("e", "e"), ("f", [0, 1, 2, 3, 4])]
async def test_query_with_merge(filled_graph_db: ArangoGraphDB, foo_model: Model) -> None: query = parse_query( '(merge_with_ancestors="foo as foobar,bar"): is("bla")') async with await filled_graph_db.search_list(QueryModel(query, foo_model) ) as cursor: async for bla in cursor: js = AccessJson(bla) assert "bar" in js.reported # key exists assert "bar" in js.desired # key exists assert "bar" in js.metadata # key exists assert js.reported.bar.is_none # bla is not a parent of this node assert js.desired.bar.is_none # bla is not a parent of this node assert js.metadata.bar.is_none # bla is not a parent of this node assert js.reported.foobar is not None # foobar is merged into reported assert js.desired.foobar is not None # foobar is merged into reported assert js.metadata.foobar is not None # foobar is merged into reported # make sure the correct parent is merged (foobar(1) -> bla(1_xxx)) assert js.reported.identifier.startswith( js.reported.foobar.identifier) assert js.reported.identifier.startswith(js.desired.foobar.node_id) assert js.reported.identifier.startswith( js.metadata.foobar.node_id)
async def test_cli(core_client: ApiClient) -> None: # make sure we have a clean slate with suppress(Exception): core_client.delete_graph(g) core_client.create_graph(g) graph_update = graph_to_json(create_graph("test")) core_client.merge_graph(graph_update, g) # evaluate search with count result = core_client.cli_evaluate("search all | count kind", g) assert len(result) == 1 parsed, to_execute = result[0] assert len(parsed.commands) == 2 assert (parsed.commands[0].cmd, parsed.commands[1].cmd) == ("search", "count") assert len(to_execute) == 2 assert (to_execute[0].get("cmd"), to_execute[1].get("cmd")) == ("execute_search", "aggregate_to_count") # execute search with count executed = list(core_client.cli_execute("search is(foo) or is(bla) | count kind", g)) assert executed == ["cloud: 1", "foo: 11", "bla: 100", "total matched: 112", "total unmatched: 0"] # list all cli commands info = AccessJson(core_client.cli_info()) assert len(info.commands) == 37
async def test_graph_api(core_client: ApiClient) -> None: # make sure we have a clean slate with suppress(Exception): core_client.delete_graph(g) # create a new graph graph = AccessJson(core_client.create_graph(g)) assert graph.id == "root" assert graph.reported.kind == "graph_root" # list all graphs graphs = core_client.list_graphs() assert g in graphs # get one specific graph graph: AccessJson = AccessJson(core_client.get_graph(g)) # type: ignore assert graph.id == "root" assert graph.reported.kind == "graph_root" # wipe the data in the graph assert core_client.delete_graph(g, truncate=True) == "Graph truncated." assert g in core_client.list_graphs() # create a node in the graph uid = rnd_str() node = AccessJson(core_client.create_node("root", uid, {"identifier": uid, "kind": "child", "name": "max"}, g)) assert node.id == uid assert node.reported.name == "max" # update a node in the graph node = AccessJson(core_client.patch_node(uid, {"name": "moritz"}, "reported", g)) assert node.id == uid assert node.reported.name == "moritz" # get the node node = AccessJson(core_client.get_node(uid, g)) assert node.id == uid assert node.reported.name == "moritz" # delete the node core_client.delete_node(uid, g) with pytest.raises(AttributeError): # node can not be found core_client.get_node(uid, g) # merge a complete graph merged = core_client.merge_graph(graph_to_json(create_graph("test")), g) assert merged == rc.GraphUpdate(112, 1, 0, 212, 0, 0) # batch graph update and commit batch1_id, batch1_info = core_client.add_to_batch(graph_to_json(create_graph("hello")), "batch1", g) assert batch1_info == rc.GraphUpdate(0, 100, 0, 0, 0, 0) assert batch1_id == "batch1" batch_infos = AccessJson.wrap_list(core_client.list_batches(g)) assert len(batch_infos) == 1 # assert batch_infos[0].id == batch1_id assert batch_infos[0].affected_nodes == ["collector"] # replace node assert batch_infos[0].is_batch is True core_client.commit_batch(batch1_id, g) # batch graph update and abort batch2_id, batch2_info = core_client.add_to_batch(graph_to_json(create_graph("bonjour")), "batch2", g) assert batch2_info == rc.GraphUpdate(0, 100, 0, 0, 0, 0) assert batch2_id == "batch2" core_client.abort_batch(batch2_id, g) # update nodes update = [{"id": node["id"], "reported": {"name": "bruce"}} for _, node in create_graph("foo").nodes(data=True)] updated_nodes = core_client.patch_nodes(update, g) assert len(updated_nodes) == 113 for n in updated_nodes: assert n.get("reported", {}).get("name") == "bruce" # create the raw search raw = core_client.search_graph_raw('id("3")', g) assert raw == { "query": "LET filter0 = (FOR m0 in graphtest FILTER m0._key == @b0 RETURN m0) " 'FOR result in filter0 RETURN UNSET(result, ["flat"])', "bind_vars": {"b0": "3"}, } # estimate the search cost = core_client.search_graph_explain('id("3")', g) assert cost.full_collection_scan is False assert cost.rating == rc.EstimatedQueryCostRating.simple # search list result_list = list(core_client.search_list('id("3") -[0:]->', graph=g)) assert len(result_list) == 11 # one parent node and 10 child nodes assert result_list[0].get("id") == "3" # first node is the parent node # search graph result_graph = list(core_client.search_graph('id("3") -[0:]->', graph=g)) assert len(result_graph) == 21 # 11 nodes + 10 edges assert result_list[0].get("id") == "3" # first node is the parent node # aggregate result_aggregate = core_client.search_aggregate("aggregate(reported.kind as kind: sum(1) as count): all", g) assert {r["group"]["kind"]: r["count"] for r in result_aggregate} == { "bla": 100, "cloud": 1, "foo": 11, "graph_root": 1, } # delete the graph assert core_client.delete_graph(g) == "Graph deleted." assert g not in core_client.list_graphs()
async def test_http_command( cli: CLI, echo_http_server: Tuple[int, List[Tuple[Request, Json]]]) -> None: port, requests = echo_http_server def test_arg( arg_str: str, method: Optional[str] = None, url: Optional[str] = None, headers: Optional[Dict[str, str]] = None, params: Optional[Dict[str, str]] = None, timeout: Optional[ClientTimeout] = None, compress: Optional[bool] = None, ) -> None: def test_if_set(prop: Any, value: Any) -> None: if prop is not None: assert prop == value, f"{prop} is not {value}" arg = HttpCommand.parse_args("https", arg_str) test_if_set(method, arg.method) test_if_set(url, arg.url) test_if_set(headers, arg.headers) test_if_set(params, arg.params) test_if_set(compress, arg.compress) test_if_set(timeout, arg.timeout) test_arg(":123", "POST", "https://localhost:123", {}, {}, ClientTimeout(30), False) test_arg("GET :123", "GET", "https://localhost:123") test_arg("://foo:123", "POST", "https://foo:123") test_arg("foo:123/bla", "POST", "https://foo:123/bla") test_arg("foo:123/bla", "POST", "https://foo:123/bla") test_arg("foo/bla", "POST", "https://foo/bla") test_arg( '--compress --timeout 24 POST :123 "hdr1: test" qp==123 hdr2:fest "qp2 == 321"', headers={ "hdr1": "test", "hdr2": "fest" }, params={ "qp": "123", "qp2": "321" }, compress=True, timeout=ClientTimeout(24), ) # take 3 instance of type bla and send it to the echo server result = await cli.execute_cli_command( f"search is(bla) limit 3 | http :{port}/test", stream.list) # one line is returned to the user with a summary of the response types. assert result == [["3 requests with status 200 sent."]] # make sure all 3 requests have been received - the body is the complete json node assert len(requests) == 3 for ar in (AccessJson(content) for _, content in requests): assert is_node(ar) assert ar.reported.kind == "bla" # failing requests are retried requests.clear() await cli.execute_cli_command( f"search is(bla) limit 1 | http --backoff-base 0.001 :{port}/fail", stream.list) # 1 request + 3 retries => 4 requests assert len(requests) == 4
async def test_tag_command(cli: CLI, performed_by: Dict[str, List[str]], incoming_tasks: List[WorkerTask], caplog: LogCaptureFixture) -> None: counter = 0 def nr_of_performed() -> int: nonlocal counter performed = len(performed_by) increase = performed - counter counter = performed return increase nr_of_performed() # reset to 0 assert await cli.execute_cli_command( "echo id_does_not_exist | tag update foo bla", stream.list) == [[]] assert nr_of_performed() == 0 res1 = await cli.execute_cli_command( 'json ["root", "collector"] | tag update foo "bla_{reported.some_int}"', stream.list) assert nr_of_performed() == 2 assert {a["id"] for a in res1[0]} == {"root", "collector"} assert len(incoming_tasks) == 2 # check that the worker task data is correct data = AccessJson(incoming_tasks[0].data) assert data["update"] is not None # tag update -> data.update is defined assert not data.node.reported.is_none # the node reported section is defined assert not data.node.metadata.is_none # the node metadata section is defined assert not data.node.ancestors.cloud.reported.is_none # the ancestors cloud section is defineda assert data[ "update"].foo == "bla_0" # using the renderer bla_{reported.some_int} res2 = await cli.execute_cli_command( 'search is("foo") | tag update foo bla', stream.list) assert nr_of_performed() == 11 assert len(res2[0]) == 11 res2_tag_no_val = await cli.execute_cli_command( 'search is("foo") | tag update foobar', stream.list) assert nr_of_performed() == 11 assert len(res2_tag_no_val[0]) == 11 res3 = await cli.execute_cli_command('search is("foo") | tag delete foo', stream.list) assert nr_of_performed() == 11 assert len(res3[0]) == 11 with caplog.at_level(logging.WARNING): caplog.clear() res4 = await cli.execute_cli_command( 'search is("bla") limit 2 | tag delete foo', stream.list) assert nr_of_performed() == 2 assert len(res4[0]) == 2 # make sure that 2 warnings are emitted assert len(caplog.records) == 2 for res in caplog.records: assert res.message.startswith( "Tag update not reflected in db. Wait until next collector run." ) # tag updates can be put into background res6 = await cli.execute_cli_command( 'json ["root", "collector"] | tag update --nowait foo bla', stream.list) assert cli.dependencies.forked_tasks.qsize() == 2 for res in res6[0]: # in this case a message with the task id is emitted assert res.startswith("Spawned WorkerTask tag:") # type:ignore # and the real result is found when the forked task is awaited, which happens by the CLI reaper awaitable, info = await cli.dependencies.forked_tasks.get() assert (await awaitable)["id"] in ["root", "collector"] # type:ignore
async def create_graph(self, name: str) -> AccessJson: async with self.session.post(self.base_path + f"/graph/{name}") as response: # root node return AccessJson(await response.json())
async def get_graph(self, name: str) -> Optional[AccessJson]: async with self.session.post(self.base_path + f"/graph/{name}") as response: return AccessJson( await response.json()) if response.status == 200 else None