async def merge_graph(self, db: DbAccess) -> GraphUpdate: # type: ignore
model = Model.from_kinds([kind async for kind in db.model_db.all()])
builder = GraphBuilder(model)
nxt = self.next_action()
while isinstance(nxt, ReadElement):
for element in nxt.jsons():
builder.add_from_json(element)
log.debug(f"Read {int(BatchSize / 1000)}K elements in process")
nxt = self.next_action()
if isinstance(nxt, PoisonPill):
log.debug("Got poison pill - going to die.")
shutdown_process(0)
elif isinstance(nxt, MergeGraph):
log.debug("Graph read into memory")
builder.check_complete()
graphdb = db.get_graph_db(nxt.graph)
outer_edge_db = db.get_pending_outer_edge_db()
_, result = await graphdb.merge_graph(builder.graph, model,
nxt.change_id, nxt.is_batch)
if nxt.task_id and builder.deferred_edges:
await outer_edge_db.update(
PendingDeferredEdges(nxt.task_id, nxt.graph,
builder.deferred_edges))
log.debug(
f"Updated {len(builder.deferred_edges)} pending outer edges for collect task {nxt.task_id}"
)
return result
async def model(self) -> Model:
async with self.session.get(self.base_path + "/model") as response:
model_json = await response.json()
model = Model.from_kinds([
from_js(kind, Kind) for kind in model_json["kinds"].values()
]) # type: ignore
return model
async def load_model(self) -> Model:
if self.__loaded_model:
return self.__loaded_model
else:
kinds = [kind async for kind in self.db.all()]
model = Model.from_kinds(list(kinds))
self.__loaded_model = model
return model
async def update_configs_model(self, update: List[Kind]) -> Model:
async with self.session.patch(self.base_path + "/configs/model",
json=to_js(update)) as response:
model_json = await response.json()
model = Model.from_kinds([
from_js(kind, Kind) for kind in model_json["kinds"].values()
]) # type: ignore
return model
def test_any() -> None:
model = Model.from_kinds(predefined_kinds)
assert model.check_valid({
"kind": "any",
"a": True,
"b": 12,
"c": [],
"d": {
"a": "b"
}
}) is None
async def get_configs_model(self) -> Model:
async with self.session.get(self.base_path + f"/configs/model") as r:
if r.status == 200:
model_json = await r.json()
model = Model.from_kinds([
from_js(kind, Kind)
for kind in model_json["kinds"].values()
]) # type: ignore
return model
else:
raise AttributeError(await r.text())
def test_dictionary() -> None:
model = {k.fqn: k for k in predefined_kinds}
result = Property.parse_kind("dictionary[string, string]", model)
assert isinstance(result, DictionaryKind)
assert result.key_kind is model["string"]
assert result.value_kind is model["string"]
result = Property.parse_kind(
"dictionary[string, dictionary[string, float]]", model)
assert isinstance(result, DictionaryKind)
assert result.key_kind is model["string"]
assert result.value_kind == DictionaryKind(model["string"], model["float"])
address = ComplexKind("Foo", [], [
Property("tags", "dictionary[string, string]"),
Property("anything", "dictionary[string, any]")
])
address_model = Model.from_kinds([address])
assert address_model.check_valid({
"kind": "Foo",
"tags": {
"a": "b",
"b": "c"
}
}) is None
expected = 'Kind:Foo Property:tags is not valid: value of dictionary[string, string] is not valid: Expected type string but got int: {"kind": "Foo", "tags": {"a": 1, "b": "c"}}'
assert expect_error(address_model, {
"kind": "Foo",
"tags": {
"a": 1,
"b": "c"
}
}) == expected
assert address_model.check_valid({
"kind": "Foo",
"anything": {
"a": 1,
"b": "c",
"c": True
}
}) is None
expected = 'Kind:Foo Property:anything is not valid: dictionary requires a json object, but got this: 1: {"kind": "Foo", "anything": 1}'
assert expect_error(address_model, {
"kind": "Foo",
"anything": 1
}) == expected
def test_load(model_json: str) -> None:
kinds: List[Kind] = [from_js(a, Kind)
for a in json.loads(model_json)] # type: ignore
model = Model.from_kinds(kinds)
assert model.check_valid({
"kind": "test.EC2",
"id": "e1",
"name": "e1",
"cores": 1,
"mem": 32,
"tags": {}
}) is None
base: ComplexKind = model["test.Base"] # type: ignore
ec2: ComplexKind = model["test.EC2"] # type: ignore
assert ec2.kind_hierarchy() == {
"test.Compound", "test.BaseResource", "test.Base", "test.EC2"
}
assert ec2.allow_unknown_props is True
assert base.allow_unknown_props is False
def test_array() -> None:
foo = ComplexKind("Foo", [], [
Property("tags", "dictionary[string, string]"),
Property("kind", "string")
])
complex_kind = ComplexKind(
"TestArray",
[],
[
Property("kind", "string"),
Property("los", "string[]"),
Property("lod", "dictionary[string, string][]"),
Property("foos", "Foo[]"),
Property("los_los", "string[][]"),
Property("los_los_los", "string[][][]"),
],
)
model = Model.from_kinds([foo, complex_kind])
assert (model.check_valid({
"kind":
"TestArray",
"los": ["a", "b", "c"],
"lod": [{
"a": "b"
}, {
"b": "c"
}],
"foos": [{
"kind": "Foo",
"tags": {
"a": "b"
}
}, {
"kind": "Foo",
"tags": {
"b": "c"
}
}],
"los_los": [["a", "b"], ["c"], ["d", "e"]],
"los_los_los": [[["a", "b"], ["c"]], [["d", "e"], ["f"]]],
}) is None)
def foo_model(foo_kinds: List[Kind]) -> Model:
return Model.from_kinds(foo_kinds)
def person_model() -> Model:
zip = StringKind("zip")
base = ComplexKind(
"Base",
[],
[
Property(
"id", "string", required=True, description="Some identifier"),
Property("kind",
"string",
required=True,
description="Kind if this node."),
Property("list", "string[]", description="A list of strings."),
Property("tags",
"dictionary[string, string]",
description="Key/value pairs."),
Property("mtime",
"datetime",
description="Modification time of this node."),
],
)
address = ComplexKind(
"Address",
["Base"],
[
Property("zip", "zip", description="The zip code."),
Property("city",
"string",
required=True,
description="The name of the city.\nAnd another line."),
],
)
person = ComplexKind(
"Person",
["Base"],
[
Property("name", "string", description="The name of the person."),
Property("address",
"Address",
description="The address of the person."),
Property("other_addresses",
"dictionary[string, Address]",
description="Other addresses."),
Property("addresses",
"Address[]",
description="The list of addresses."),
Property("any", "any", description="Some arbitrary value."),
],
)
any_foo = ComplexKind(
"any_foo",
["Base"],
[
Property("foo", "any", description="Some foo value."),
Property("test", "string", description="Some test value."),
],
)
cloud = ComplexKind("cloud", ["Base"], [])
account = ComplexKind("account", ["Base"], [])
region = ComplexKind("region", ["Base"], [])
parent = ComplexKind("parent", ["Base"], [])
child = ComplexKind("child", ["Base"], [])
return Model.from_kinds([
zip, person, address, base, any_foo, cloud, account, region, parent,
child
])
async def update_model(self, kinds: List[Kind]) -> Model:
self.model = Model.from_kinds(kinds)
return self.model
async def get_configs_model(self) -> Model:
kinds = [kind async for kind in self.model_db.all()]
return Model.from_kinds(list(kinds))