def test_property_path_on_model(person_model: Model) -> None:
# complex based property path
person: ComplexKind = cast(ComplexKind, person_model["Person"])
person_path = {p.path: p for p in person.resolved_properties()}
assert len(person_path) == 13
assert person_path[PropertyPath(["name"])].kind == person_model["string"]
assert person_path[PropertyPath(["name"])].prop.name == "name"
assert person_path[PropertyPath(["list[]"])].kind == person_model["string"]
assert person_path[PropertyPath(["list[]"])].prop.name == "list"
assert person_path[PropertyPath(["tags",
None])].kind == person_model["string"]
assert person_path[PropertyPath(["address",
"zip"])].kind == person_model["zip"]
assert person_path[PropertyPath(["address", "zip"])].prop.name == "zip"
with pytest.raises(KeyError):
_ = person_path[PropertyPath(["anything"])]
# model based property path
assert person_model.kind_by_path("name") == person_model["string"]
assert person_model.kind_by_path("list[]") == person_model["string"]
assert person_model.kind_by_path("tags.foo") == person_model["string"]
assert person_model.kind_by_path("tags.bla") == person_model["string"]
assert person_model.kind_by_path(
"other_addresses.bla.zip") == person_model["zip"]
assert person_model.kind_by_path("address.zip") == person_model["zip"]
def test_update(person_model: Model) -> None:
with pytest.raises(
AttributeError) as not_allowed: # update city with different type
person_model.update_kinds([
ComplexKind(
"Address",
["Base"],
[
Property("city", "int32", required=True),
],
)
])
assert (
str(not_allowed.value) ==
"Update not possible: following properties would be non unique having the same path but different type: "
"Address.city (string -> int32)")
updated = person_model.update_kinds([StringKind("Foo")])
assert updated["Foo"].fqn == "Foo"
with pytest.raises(AttributeError) as simple:
updated.update_kinds([ComplexKind("Foo", [], [])])
assert str(
simple.value) == "Update Foo changes an existing property type Foo"
with pytest.raises(AttributeError) as duplicate:
updated.update_kinds(
[ComplexKind("Bla", [], [Property("id", "int32")])])
assert (
str(duplicate.value) ==
"Update not possible: following properties would be non unique having the same path but different type: "
"Bla.id (string -> int32)")
async def merge_graph(self, db: DbAccess) -> GraphUpdate: # type: ignore
model = Model.from_kinds([kind async for kind in db.model_db.all()])
builder = GraphBuilder(model)
nxt = self.next_action()
while isinstance(nxt, ReadElement):
for element in nxt.jsons():
builder.add_from_json(element)
log.debug(f"Read {int(BatchSize / 1000)}K elements in process")
nxt = self.next_action()
if isinstance(nxt, PoisonPill):
log.debug("Got poison pill - going to die.")
shutdown_process(0)
elif isinstance(nxt, MergeGraph):
log.debug("Graph read into memory")
builder.check_complete()
graphdb = db.get_graph_db(nxt.graph)
outer_edge_db = db.get_pending_outer_edge_db()
_, result = await graphdb.merge_graph(builder.graph, model,
nxt.change_id, nxt.is_batch)
if nxt.task_id and builder.deferred_edges:
await outer_edge_db.update(
PendingDeferredEdges(nxt.task_id, nxt.graph,
builder.deferred_edges))
log.debug(
f"Updated {len(builder.deferred_edges)} pending outer edges for collect task {nxt.task_id}"
)
return result
async def model(self) -> Model:
async with self.session.get(self.base_path + "/model") as response:
model_json = await response.json()
model = Model.from_kinds([
from_js(kind, Kind) for kind in model_json["kinds"].values()
]) # type: ignore
return model
async def load_model(self) -> Model:
if self.__loaded_model:
return self.__loaded_model
else:
kinds = [kind async for kind in self.db.all()]
model = Model.from_kinds(list(kinds))
self.__loaded_model = model
return model
async def update_configs_model(self, update: List[Kind]) -> Model:
async with self.session.patch(self.base_path + "/configs/model",
json=to_js(update)) as response:
model_json = await response.json()
model = Model.from_kinds([
from_js(kind, Kind) for kind in model_json["kinds"].values()
]) # type: ignore
return model
def test_ip_range() -> None:
bind_vars: Json = {}
model = QueryModel(
Query.by(IsTerm(["foo"])).on_section("reported"), Model.empty())
result = in_subnet(
"crs", bind_vars,
FunctionTerm("in_subnet", "foo.bla", ["192.168.1.0/24"]), model)
assert result == "BIT_AND(IPV4_TO_NUMBER(crs.foo.bla), 4294967040) == @0"
assert bind_vars["0"] == 3232235776
async def get_configs_model(self) -> Model:
async with self.session.get(self.base_path + f"/configs/model") as r:
if r.status == 200:
model_json = await r.json()
model = Model.from_kinds([
from_js(kind, Kind)
for kind in model_json["kinds"].values()
]) # type: ignore
return model
else:
raise AttributeError(await r.text())
def test_any() -> None:
model = Model.from_kinds(predefined_kinds)
assert model.check_valid({
"kind": "any",
"a": True,
"b": 12,
"c": [],
"d": {
"a": "b"
}
}) is None
def test_has_key() -> None:
bind_vars: Json = {}
model = QueryModel(Query.by("foo"), Model.empty())
result = has_key("crs", bind_vars,
FunctionTerm("has_key", "foo.bla", [["a", "b", "c"]]),
model)
assert result == "@fn0 ALL IN ATTRIBUTES(crs.foo.bla, true)"
assert bind_vars["fn0"] == ["a", "b", "c"]
bind_vars2: Json = {}
result = has_key("crs", bind_vars2,
FunctionTerm("has_key", "foo.bla", ["a"]), model)
assert result == "HAS(crs.foo.bla, @fn0)"
assert bind_vars2["fn0"] == "a"
def test_dictionary() -> None:
model = {k.fqn: k for k in predefined_kinds}
result = Property.parse_kind("dictionary[string, string]", model)
assert isinstance(result, DictionaryKind)
assert result.key_kind is model["string"]
assert result.value_kind is model["string"]
result = Property.parse_kind(
"dictionary[string, dictionary[string, float]]", model)
assert isinstance(result, DictionaryKind)
assert result.key_kind is model["string"]
assert result.value_kind == DictionaryKind(model["string"], model["float"])
address = ComplexKind("Foo", [], [
Property("tags", "dictionary[string, string]"),
Property("anything", "dictionary[string, any]")
])
address_model = Model.from_kinds([address])
assert address_model.check_valid({
"kind": "Foo",
"tags": {
"a": "b",
"b": "c"
}
}) is None
expected = 'Kind:Foo Property:tags is not valid: value of dictionary[string, string] is not valid: Expected type string but got int: {"kind": "Foo", "tags": {"a": 1, "b": "c"}}'
assert expect_error(address_model, {
"kind": "Foo",
"tags": {
"a": 1,
"b": "c"
}
}) == expected
assert address_model.check_valid({
"kind": "Foo",
"anything": {
"a": 1,
"b": "c",
"c": True
}
}) is None
expected = 'Kind:Foo Property:anything is not valid: dictionary requires a json object, but got this: 1: {"kind": "Foo", "anything": 1}'
assert expect_error(address_model, {
"kind": "Foo",
"anything": 1
}) == expected
def test_load(model_json: str) -> None:
kinds: List[Kind] = [from_js(a, Kind)
for a in json.loads(model_json)] # type: ignore
model = Model.from_kinds(kinds)
assert model.check_valid({
"kind": "test.EC2",
"id": "e1",
"name": "e1",
"cores": 1,
"mem": 32,
"tags": {}
}) is None
base: ComplexKind = model["test.Base"] # type: ignore
ec2: ComplexKind = model["test.EC2"] # type: ignore
assert ec2.kind_hierarchy() == {
"test.Compound", "test.BaseResource", "test.Base", "test.EC2"
}
assert ec2.allow_unknown_props is True
assert base.allow_unknown_props is False
def test_array() -> None:
foo = ComplexKind("Foo", [], [
Property("tags", "dictionary[string, string]"),
Property("kind", "string")
])
complex_kind = ComplexKind(
"TestArray",
[],
[
Property("kind", "string"),
Property("los", "string[]"),
Property("lod", "dictionary[string, string][]"),
Property("foos", "Foo[]"),
Property("los_los", "string[][]"),
Property("los_los_los", "string[][][]"),
],
)
model = Model.from_kinds([foo, complex_kind])
assert (model.check_valid({
"kind":
"TestArray",
"los": ["a", "b", "c"],
"lod": [{
"a": "b"
}, {
"b": "c"
}],
"foos": [{
"kind": "Foo",
"tags": {
"a": "b"
}
}, {
"kind": "Foo",
"tags": {
"b": "c"
}
}],
"los_los": [["a", "b"], ["c"], ["d", "e"]],
"los_los_los": [[["a", "b"], ["c"]], [["d", "e"], ["f"]]],
}) is None)
async def test_emit_recurrent_events() -> None:
message_bus = MessageBus()
sender = InMemoryEventSender()
model = ModelHandlerStatic(Model.empty())
sub = SubscriptionHandler(
InMemoryDb[Subscriber](Subscriber, lambda x: x.id), message_bus)
queue = WorkerTaskQueue()
fast = timedelta(seconds=0.001)
periodic = emit_recurrent_events(sender, model, sub, queue, message_bus,
fast, fast)
await periodic.start()
while len(sender.events) < 3:
await asyncio.sleep(0.01)
await periodic.stop()
model_event, subscriber_event, worker_event = sender.events[0:3]
assert model_event.kind == CoreEvent.ModelInfo
assert model_event.counters["model_count"] == 0
assert subscriber_event.kind == CoreEvent.SubscriberInfo
assert subscriber_event.counters["subscriber_count"] == 0
assert worker_event.kind == CoreEvent.WorkerQueueInfo
assert worker_event.counters["worker_count"] == 0
def person_model() -> Model:
zip = StringKind("zip")
base = ComplexKind(
"Base",
[],
[
Property(
"id", "string", required=True, description="Some identifier"),
Property("kind",
"string",
required=True,
description="Kind if this node."),
Property("list", "string[]", description="A list of strings."),
Property("tags",
"dictionary[string, string]",
description="Key/value pairs."),
Property("mtime",
"datetime",
description="Modification time of this node."),
],
)
address = ComplexKind(
"Address",
["Base"],
[
Property("zip", "zip", description="The zip code."),
Property("city",
"string",
required=True,
description="The name of the city.\nAnd another line."),
],
)
person = ComplexKind(
"Person",
["Base"],
[
Property("name", "string", description="The name of the person."),
Property("address",
"Address",
description="The address of the person."),
Property("other_addresses",
"dictionary[string, Address]",
description="Other addresses."),
Property("addresses",
"Address[]",
description="The list of addresses."),
Property("any", "any", description="Some arbitrary value."),
],
)
any_foo = ComplexKind(
"any_foo",
["Base"],
[
Property("foo", "any", description="Some foo value."),
Property("test", "string", description="Some test value."),
],
)
cloud = ComplexKind("cloud", ["Base"], [])
account = ComplexKind("account", ["Base"], [])
region = ComplexKind("region", ["Base"], [])
parent = ComplexKind("parent", ["Base"], [])
child = ComplexKind("child", ["Base"], [])
return Model.from_kinds([
zip, person, address, base, any_foo, cloud, account, region, parent,
child
])
def test_graph(person_model: Model) -> None:
graph: DiGraph = person_model.graph()
assert len(graph.nodes()) == 11
assert len(graph.edges()) == 8
async def update_model(self, kinds: List[Kind]) -> Model:
self.model = Model.from_kinds(kinds)
return self.model
def test_model_checking(person_model: Model) -> None:
assert person_model.check_valid({"kind": "Base", "id": "32"}) is None
assert person_model.check_valid({
"kind": "Base",
"id": "32",
"list": ["one", "two"]
}) is None
expected = 'Kind:Base Property:list is not valid: Expected type string but got int: {"kind": "Base", "id": "32", "list": [1, 2]}'
assert expect_error(person_model, {
"kind": "Base",
"id": "32",
"list": [1, 2]
}) == expected
expected = 'Kind:Base Property:list is not valid: Expected property is not an array!: {"kind": "Base", "id": "32", "list": "not iterable"}'
assert expect_error(person_model, {
"kind": "Base",
"id": "32",
"list": "not iterable"
}) == expected
expected = 'Kind:Base Property:id is not valid: Expected type string but got int: {"kind": "Base", "id": 32}'
assert expect_error(person_model, {"kind": "Base", "id": 32}) == expected
expected = 'Kind:Base Property:id is required and missing in {"kind": "Base"}'
assert expect_error(person_model, {"kind": "Base"}) == expected
expected = "Kind:Base Property:unknown is not defined in model!"
assert expect_error(person_model, {
"kind": "Base",
"id": "bla",
"unknown": 1
}) == expected
expected = (
'Kind:Address Property:id is required and missing in {"kind": "Address", "zip": "12345", "city": "gotham"}'
)
assert expect_error(person_model, {
"kind": "Address",
"zip": "12345",
"city": "gotham"
}) == expected
nested = {
"id": "batman",
"kind": "Person",
"name": "batman",
"address": {
"kind": "Address",
"id": "foo",
"city": "gotham"
},
}
assert person_model.check_valid(nested) is None
nested = {
"id": "batman",
"kind": "Person",
"name": "batman",
"address": {
"kind": "Address",
"city": "gotham"
}
}
expected = 'Kind:Person Property:address is not valid: Kind:Address Property:id is required and missing in {"kind": "Address", "city": "gotham"}: {"id": "batman", "kind": "Person", "name": "batman", "address": {"kind": "Address", "city": "gotham"}}'
assert expect_error(person_model, nested) == expected
assert person_model.check_valid({
"kind": "Base",
"id": "32",
"mtime": "2008-09-03T20:56:35+20:00"
})["mtime"] == "2008-09-03T00:56:35Z" # type: ignore
anything = {
"kind": "any",
"some": [1, 2, 3],
"not": "defined",
"props": True
}
assert person_model.check_valid(anything) is None
any_foo = {
"kind": "any_foo",
"id": "foo",
"foo": {
"a": [1, 2, 3]
},
"test": "hallo"
}
assert person_model.check_valid(any_foo) is None
def foo_model(foo_kinds: List[Kind]) -> Model:
return Model.from_kinds(foo_kinds)
async def get_configs_model(self) -> Model:
kinds = [kind async for kind in self.model_db.all()]
return Model.from_kinds(list(kinds))