async def test_events(event_graph_db: EventGraphDB, foo_model: Model,
event_sender: InMemoryEventSender) -> None:
await event_graph_db.create_node(foo_model, "some_other",
to_json(Foo("some_other", "foo")), "root")
await event_graph_db.update_node(foo_model, "some_other", {"name": "bla"},
False, "reported")
await event_graph_db.delete_node("some_other")
await event_graph_db.merge_graph(create_graph("yes or no", width=1),
foo_model)
await event_graph_db.merge_graph(create_graph("maybe", width=1), foo_model,
"batch1", True)
# make sure all events will arrive
await asyncio.sleep(0.1)
# ensure the correct count and order of events
assert [a.kind for a in event_sender.events] == [
CoreEvent.NodeCreated,
CoreEvent.NodeUpdated,
CoreEvent.NodeDeleted,
CoreEvent.GraphMerged,
CoreEvent.BatchUpdateGraphMerged,
]
merge_event = AccessJson(event_sender.events[3].context)
assert merge_event.graph == event_graph_db.graph_name
assert merge_event.providers == ["collector"]
assert merge_event.batch is False
async def get_node(self, graph: str, node_id: str) -> AccessJson:
async with self.session.get(
self.base_path + f"/graph/{graph}/node/{node_id}") as response:
if response.status == 200:
return AccessJson(await response.json())
else:
raise AttributeError(await response.text())
async def patch_config(self, config_id: str, json: Json) -> AccessJson:
async with self.session.patch(self.base_path + f"/config/{config_id}",
json=json) as r:
if r.status == 200:
return AccessJson.wrap_object(await r.json())
else:
raise AttributeError(await r.text())
async def list_batches(self, graph: str) -> List[AccessJson]:
async with self.session.get(self.base_path +
f"/graph/{graph}/batch") as r:
if r.status == 200:
return AccessJson.wrap_list(await r.json())
else:
raise AttributeError(await r.text())
async def search_graph(self, graph: str, search: str) -> List[AccessJson]:
async with self.session.post(self.base_path +
f"/graph/{graph}/search/graph",
data=search) as r:
if r.status == 200:
return AccessJson.wrap_list(await r.json())
else:
raise AttributeError(await r.text())
async def patch_nodes(self, graph: str,
nodes: List[Json]) -> List[AccessJson]:
async with self.session.patch(self.base_path + f"/graph/{graph}/nodes",
json=nodes) as response:
if response.status == 200:
return AccessJson.wrap_list(await response.json())
else:
raise AttributeError(await response.text())
async def cli_execute(self, graph: str, command: str,
**env: str) -> List[JsonElement]:
props = {"graph": graph, "section": "reported", **env}
async with self.session.post(self.base_path + f"/cli/execute",
data=command,
params=props) as r:
if r.status == 200:
return AccessJson.wrap_list(await r.json()) # type: ignore
else:
raise AttributeError(await r.text())
async def create_node(self, graph: str, parent_node_id: str, node_id: str,
node: Json) -> AccessJson:
async with self.session.post(
self.base_path +
f"/graph/{graph}/node/{node_id}/under/{parent_node_id}",
json=node) as response:
if response.status == 200:
return AccessJson(await response.json())
else:
raise AttributeError(await response.text())
async def put_config(self,
config_id: str,
json: Json,
validate: bool = True) -> AccessJson:
params = {"validate": "true" if validate else "false"}
async with self.session.put(self.base_path + f"/config/{config_id}",
json=json,
params=params) as r:
if r.status == 200:
return AccessJson.wrap_object(await r.json())
else:
raise AttributeError(await r.text())
async def patch_node(self,
graph: str,
node_id: str,
node: Json,
section: Optional[str] = None) -> AccessJson:
section_path = f"/section/{section}" if section else ""
async with self.session.patch(
self.base_path +
f"/graph/{graph}/node/{node_id}{section_path}",
json=node) as response:
if response.status == 200:
return AccessJson(await response.json())
else:
raise AttributeError(await response.text())
def test_resolve_graph_data() -> None:
g = multi_cloud_graph("account")
graph = GraphAccess(g)
graph.resolve()
# ancestor data should be stored in metadata
n1 = AccessJson(graph.node("child_parent_region_account_cloud_gcp_1_europe_1_0")) # type: ignore
assert n1.refs.region_id == "region_account_cloud_gcp_1_europe"
assert n1.ancestors.account.reported.id == "id_account_cloud_gcp_1"
assert n1.ancestors.account.reported.name == "name_account_cloud_gcp_1"
assert n1.ancestors.region.reported.id == "id_region_account_cloud_gcp_1_europe"
assert n1.ancestors.region.reported.name == "name_region_account_cloud_gcp_1_europe"
# make sure there is no summary
assert n1.descendant_summary == AccessNone(None)
r1 = AccessJson(graph.node("region_account_cloud_gcp_1_europe")) # type: ignore
assert r1.metadata.descendant_summary == {"child": 9}
assert r1.metadata.descendant_count == 9
r2 = AccessJson(graph.node("account_cloud_gcp_1")) # type: ignore
assert r2.metadata.descendant_summary == {"child": 54, "region": 6}
assert r2.metadata.descendant_count == 60
r3 = AccessJson(graph.node("cloud_gcp")) # type: ignore
assert r3.metadata.descendant_summary == {"child": 162, "region": 18, "account": 3}
assert r3.metadata.descendant_count == 183
async def cli_evaluate(
self, graph: str, command: str,
**env: str) -> List[Tuple[ParsedCommands, List[AccessJson]]]:
props = {"graph": graph, "section": "reported", **env}
async with self.session.post(self.base_path + f"/cli/evaluate",
data=command,
params=props) as r:
if r.status == 200:
return [(
ParsedCommands(
from_js(json["parsed"], List[ParsedCommand]),
json["env"]),
AccessJson.wrap(json["execute"]),
) for json in await r.json()]
else:
raise AttributeError(await r.text())
def test_access_json() -> None:
js = {"a": "a", "b": {"c": "c", "d": {"e": "e", "f": [0, 1, 2, 3, 4]}}}
access = AccessJson(js, "null")
assert access.a == "a"
assert access.b.d.f[2] == 2
assert str(access.foo.bla.bar[23].now) is "null"
assert json.dumps(access.b.d,
sort_keys=True) == '{"e": "e", "f": [0, 1, 2, 3, 4]}'
assert access["a"] == "a"
assert access["b"]["d"]["f"][2] == 2
assert str(access["foo"]["bla"]["bar"][23]["now"]) is "null"
assert json.dumps(access["b"]["d"],
sort_keys=True) == '{"e": "e", "f": [0, 1, 2, 3, 4]}'
assert [a for a in access] == ["a", "b"]
assert [a for a in access.doesnt.exist] == []
assert [a for a in access.b.d.items()] == [("e", "e"),
("f", [0, 1, 2, 3, 4])]
async def test_query_merge(filled_graph_db: ArangoGraphDB,
foo_model: Model) -> None:
q = parse_query("is(foo) --> is(bla) { "
"foo.bar.parents[]: <-[1:]-, "
"foo.child: -->, "
"walk: <-- -->, "
"bla.agg: aggregate(sum(1) as count): <-[0:]- "
"}")
async with await filled_graph_db.search_list(QueryModel(q, foo_model),
with_count=True) as cursor:
assert cursor.count() == 100
async for bla in cursor:
b = AccessJson(bla)
assert b.reported.kind == "bla"
assert len(b.foo.bar.parents) == 4
for parent in b.foo.bar.parents:
assert parent.reported.kind in ["foo", "cloud", "graph_root"]
assert b.walk.reported.kind == "bla"
assert b.foo.child == AccessNone()
assert b.bla.agg == [{"count": 5}]
async def test_query_with_merge(filled_graph_db: ArangoGraphDB,
foo_model: Model) -> None:
query = parse_query(
'(merge_with_ancestors="foo as foobar,bar"): is("bla")')
async with await filled_graph_db.search_list(QueryModel(query, foo_model)
) as cursor:
async for bla in cursor:
js = AccessJson(bla)
assert "bar" in js.reported # key exists
assert "bar" in js.desired # key exists
assert "bar" in js.metadata # key exists
assert js.reported.bar.is_none # bla is not a parent of this node
assert js.desired.bar.is_none # bla is not a parent of this node
assert js.metadata.bar.is_none # bla is not a parent of this node
assert js.reported.foobar is not None # foobar is merged into reported
assert js.desired.foobar is not None # foobar is merged into reported
assert js.metadata.foobar is not None # foobar is merged into reported
# make sure the correct parent is merged (foobar(1) -> bla(1_xxx))
assert js.reported.identifier.startswith(
js.reported.foobar.identifier)
assert js.reported.identifier.startswith(js.desired.foobar.node_id)
assert js.reported.identifier.startswith(
js.metadata.foobar.node_id)
async def test_cli(core_client: ApiClient) -> None:
# make sure we have a clean slate
with suppress(Exception):
core_client.delete_graph(g)
core_client.create_graph(g)
graph_update = graph_to_json(create_graph("test"))
core_client.merge_graph(graph_update, g)
# evaluate search with count
result = core_client.cli_evaluate("search all | count kind", g)
assert len(result) == 1
parsed, to_execute = result[0]
assert len(parsed.commands) == 2
assert (parsed.commands[0].cmd, parsed.commands[1].cmd) == ("search", "count")
assert len(to_execute) == 2
assert (to_execute[0].get("cmd"), to_execute[1].get("cmd")) == ("execute_search", "aggregate_to_count")
# execute search with count
executed = list(core_client.cli_execute("search is(foo) or is(bla) | count kind", g))
assert executed == ["cloud: 1", "foo: 11", "bla: 100", "total matched: 112", "total unmatched: 0"]
# list all cli commands
info = AccessJson(core_client.cli_info())
assert len(info.commands) == 37
def format_object(obj: Any) -> str:
return formatter.format_map(
AccessJson.wrap(obj, "null", render_simple_property))
async def cli_info(self) -> AccessJson:
async with self.session.get(self.base_path + f"/cli/info") as r:
if r.status == 200:
return AccessJson.wrap_object(await r.json())
else:
raise AttributeError(await r.text())
async def test_update_nodes(graph_db: ArangoGraphDB, foo_model: Model) -> None:
def expect(jsons: List[Json], path: List[str], value: JsonElement) -> None:
for js in jsons:
v = value_in_path(js, path)
assert v is not None
assert v == value
await graph_db.wipe()
await graph_db.create_node(foo_model, "id1", to_json(Foo("id1", "foo")),
"root")
await graph_db.create_node(foo_model, "id2", to_json(Foo("id2", "foo")),
"root")
# only change the desired section
change1 = {"desired": {"test": True}}
result1 = [
a async for a in graph_db.update_nodes(foo_model, {
"id1": change1,
"id2": change1
})
]
assert len(result1) == 2
expect(result1, ["desired", "test"], True)
# only change the metadata section
change2 = {"metadata": {"test": True}}
result2 = [
a async for a in graph_db.update_nodes(foo_model, {
"id1": change2,
"id2": change2
})
]
assert len(result2) == 2
expect(result2, ["metadata", "test"], True)
# change all sections including the reported section
change3 = {
"desired": {
"test": False
},
"metadata": {
"test": False
},
"reported": {
"name": "test"
}
}
node_raw_id1 = AccessJson.wrap_object(
graph_db.db.db.collection(graph_db.name).get("id1"))
result3 = [
a async for a in graph_db.update_nodes(foo_model, {
"id1": change3,
"id2": change3
})
]
assert len(result3) == 2
expect(result3, ["desired", "test"], False)
expect(result3, ["metadata", "test"], False)
expect(result3, ["reported", "name"], "test")
# make sure the db is updated
node_raw_id1_updated = AccessJson.wrap_object(
graph_db.db.db.collection(graph_db.name).get("id1"))
assert node_raw_id1.reported.name != node_raw_id1_updated.reported.name
assert node_raw_id1.desired.test != node_raw_id1_updated.desired.test
assert node_raw_id1.metadata.test != node_raw_id1_updated.metadata.test
assert node_raw_id1.flat != node_raw_id1_updated.flat
assert node_raw_id1.hash != node_raw_id1_updated.hash
assert "test" in node_raw_id1_updated.flat
change4 = {"desired": None, "metadata": None}
result4 = [
a async for a in graph_db.update_nodes(foo_model, {
"id1": change4,
"id2": change4
})
]
assert len(result4) == 2
assert "desired" not in result4
assert "metadata" not in result4
async def configs(self) -> List[str]:
async with self.session.get(self.base_path + f"/configs") as r:
if r.status == 200:
return AccessJson.wrap_list(await r.json()) # type: ignore
else:
raise AttributeError(await r.text())
async def get_graph(self, name: str) -> Optional[AccessJson]:
async with self.session.post(self.base_path +
f"/graph/{name}") as response:
return AccessJson(
await response.json()) if response.status == 200 else None
async def test_graph_api(core_client: ApiClient) -> None:
# make sure we have a clean slate
with suppress(Exception):
core_client.delete_graph(g)
# create a new graph
graph = AccessJson(core_client.create_graph(g))
assert graph.id == "root"
assert graph.reported.kind == "graph_root"
# list all graphs
graphs = core_client.list_graphs()
assert g in graphs
# get one specific graph
graph: AccessJson = AccessJson(core_client.get_graph(g)) # type: ignore
assert graph.id == "root"
assert graph.reported.kind == "graph_root"
# wipe the data in the graph
assert core_client.delete_graph(g, truncate=True) == "Graph truncated."
assert g in core_client.list_graphs()
# create a node in the graph
uid = rnd_str()
node = AccessJson(core_client.create_node("root", uid, {"identifier": uid, "kind": "child", "name": "max"}, g))
assert node.id == uid
assert node.reported.name == "max"
# update a node in the graph
node = AccessJson(core_client.patch_node(uid, {"name": "moritz"}, "reported", g))
assert node.id == uid
assert node.reported.name == "moritz"
# get the node
node = AccessJson(core_client.get_node(uid, g))
assert node.id == uid
assert node.reported.name == "moritz"
# delete the node
core_client.delete_node(uid, g)
with pytest.raises(AttributeError):
# node can not be found
core_client.get_node(uid, g)
# merge a complete graph
merged = core_client.merge_graph(graph_to_json(create_graph("test")), g)
assert merged == rc.GraphUpdate(112, 1, 0, 212, 0, 0)
# batch graph update and commit
batch1_id, batch1_info = core_client.add_to_batch(graph_to_json(create_graph("hello")), "batch1", g)
assert batch1_info == rc.GraphUpdate(0, 100, 0, 0, 0, 0)
assert batch1_id == "batch1"
batch_infos = AccessJson.wrap_list(core_client.list_batches(g))
assert len(batch_infos) == 1
# assert batch_infos[0].id == batch1_id
assert batch_infos[0].affected_nodes == ["collector"] # replace node
assert batch_infos[0].is_batch is True
core_client.commit_batch(batch1_id, g)
# batch graph update and abort
batch2_id, batch2_info = core_client.add_to_batch(graph_to_json(create_graph("bonjour")), "batch2", g)
assert batch2_info == rc.GraphUpdate(0, 100, 0, 0, 0, 0)
assert batch2_id == "batch2"
core_client.abort_batch(batch2_id, g)
# update nodes
update = [{"id": node["id"], "reported": {"name": "bruce"}} for _, node in create_graph("foo").nodes(data=True)]
updated_nodes = core_client.patch_nodes(update, g)
assert len(updated_nodes) == 113
for n in updated_nodes:
assert n.get("reported", {}).get("name") == "bruce"
# create the raw search
raw = core_client.search_graph_raw('id("3")', g)
assert raw == {
"query": "LET filter0 = (FOR m0 in graphtest FILTER m0._key == @b0 RETURN m0) "
'FOR result in filter0 RETURN UNSET(result, ["flat"])',
"bind_vars": {"b0": "3"},
}
# estimate the search
cost = core_client.search_graph_explain('id("3")', g)
assert cost.full_collection_scan is False
assert cost.rating == rc.EstimatedQueryCostRating.simple
# search list
result_list = list(core_client.search_list('id("3") -[0:]->', graph=g))
assert len(result_list) == 11 # one parent node and 10 child nodes
assert result_list[0].get("id") == "3" # first node is the parent node
# search graph
result_graph = list(core_client.search_graph('id("3") -[0:]->', graph=g))
assert len(result_graph) == 21 # 11 nodes + 10 edges
assert result_list[0].get("id") == "3" # first node is the parent node
# aggregate
result_aggregate = core_client.search_aggregate("aggregate(reported.kind as kind: sum(1) as count): all", g)
assert {r["group"]["kind"]: r["count"] for r in result_aggregate} == {
"bla": 100,
"cloud": 1,
"foo": 11,
"graph_root": 1,
}
# delete the graph
assert core_client.delete_graph(g) == "Graph deleted."
assert g not in core_client.list_graphs()
async def test_http_command(
cli: CLI, echo_http_server: Tuple[int, List[Tuple[Request,
Json]]]) -> None:
port, requests = echo_http_server
def test_arg(
arg_str: str,
method: Optional[str] = None,
url: Optional[str] = None,
headers: Optional[Dict[str, str]] = None,
params: Optional[Dict[str, str]] = None,
timeout: Optional[ClientTimeout] = None,
compress: Optional[bool] = None,
) -> None:
def test_if_set(prop: Any, value: Any) -> None:
if prop is not None:
assert prop == value, f"{prop} is not {value}"
arg = HttpCommand.parse_args("https", arg_str)
test_if_set(method, arg.method)
test_if_set(url, arg.url)
test_if_set(headers, arg.headers)
test_if_set(params, arg.params)
test_if_set(compress, arg.compress)
test_if_set(timeout, arg.timeout)
test_arg(":123", "POST", "https://localhost:123", {}, {},
ClientTimeout(30), False)
test_arg("GET :123", "GET", "https://localhost:123")
test_arg("://foo:123", "POST", "https://foo:123")
test_arg("foo:123/bla", "POST", "https://foo:123/bla")
test_arg("foo:123/bla", "POST", "https://foo:123/bla")
test_arg("foo/bla", "POST", "https://foo/bla")
test_arg(
'--compress --timeout 24 POST :123 "hdr1: test" qp==123 hdr2:fest "qp2 == 321"',
headers={
"hdr1": "test",
"hdr2": "fest"
},
params={
"qp": "123",
"qp2": "321"
},
compress=True,
timeout=ClientTimeout(24),
)
# take 3 instance of type bla and send it to the echo server
result = await cli.execute_cli_command(
f"search is(bla) limit 3 | http :{port}/test", stream.list)
# one line is returned to the user with a summary of the response types.
assert result == [["3 requests with status 200 sent."]]
# make sure all 3 requests have been received - the body is the complete json node
assert len(requests) == 3
for ar in (AccessJson(content) for _, content in requests):
assert is_node(ar)
assert ar.reported.kind == "bla"
# failing requests are retried
requests.clear()
await cli.execute_cli_command(
f"search is(bla) limit 1 | http --backoff-base 0.001 :{port}/fail",
stream.list)
# 1 request + 3 retries => 4 requests
assert len(requests) == 4
async def test_system_info_command(cli: CLI) -> None:
info = AccessJson.wrap_object(
(await cli.execute_cli_command("system info", stream.list))[0][0])
assert info.version == version()
assert info.name == "resotocore"
assert info.cpus > 0
async def test_tag_command(cli: CLI, performed_by: Dict[str, List[str]],
incoming_tasks: List[WorkerTask],
caplog: LogCaptureFixture) -> None:
counter = 0
def nr_of_performed() -> int:
nonlocal counter
performed = len(performed_by)
increase = performed - counter
counter = performed
return increase
nr_of_performed() # reset to 0
assert await cli.execute_cli_command(
"echo id_does_not_exist | tag update foo bla", stream.list) == [[]]
assert nr_of_performed() == 0
res1 = await cli.execute_cli_command(
'json ["root", "collector"] | tag update foo "bla_{reported.some_int}"',
stream.list)
assert nr_of_performed() == 2
assert {a["id"] for a in res1[0]} == {"root", "collector"}
assert len(incoming_tasks) == 2
# check that the worker task data is correct
data = AccessJson(incoming_tasks[0].data)
assert data["update"] is not None # tag update -> data.update is defined
assert not data.node.reported.is_none # the node reported section is defined
assert not data.node.metadata.is_none # the node metadata section is defined
assert not data.node.ancestors.cloud.reported.is_none # the ancestors cloud section is defineda
assert data[
"update"].foo == "bla_0" # using the renderer bla_{reported.some_int}
res2 = await cli.execute_cli_command(
'search is("foo") | tag update foo bla', stream.list)
assert nr_of_performed() == 11
assert len(res2[0]) == 11
res2_tag_no_val = await cli.execute_cli_command(
'search is("foo") | tag update foobar', stream.list)
assert nr_of_performed() == 11
assert len(res2_tag_no_val[0]) == 11
res3 = await cli.execute_cli_command('search is("foo") | tag delete foo',
stream.list)
assert nr_of_performed() == 11
assert len(res3[0]) == 11
with caplog.at_level(logging.WARNING):
caplog.clear()
res4 = await cli.execute_cli_command(
'search is("bla") limit 2 | tag delete foo', stream.list)
assert nr_of_performed() == 2
assert len(res4[0]) == 2
# make sure that 2 warnings are emitted
assert len(caplog.records) == 2
for res in caplog.records:
assert res.message.startswith(
"Tag update not reflected in db. Wait until next collector run."
)
# tag updates can be put into background
res6 = await cli.execute_cli_command(
'json ["root", "collector"] | tag update --nowait foo bla',
stream.list)
assert cli.dependencies.forked_tasks.qsize() == 2
for res in res6[0]:
# in this case a message with the task id is emitted
assert res.startswith("Spawned WorkerTask tag:") # type:ignore
# and the real result is found when the forked task is awaited, which happens by the CLI reaper
awaitable, info = await cli.dependencies.forked_tasks.get()
assert (await awaitable)["id"] in ["root", "collector"] # type:ignore
async def create_graph(self, name: str) -> AccessJson:
async with self.session.post(self.base_path +
f"/graph/{name}") as response:
# root node
return AccessJson(await response.json())