def count_successors_by(node_id: str, edge_type: str,
path: List[str]) -> Dict[str, int]:
result: Dict[str, int] = {}
to_visit = list(self.successors(node_id, edge_type))
while to_visit:
visit_next: List[str] = []
for elem_id in to_visit:
if elem_id not in visited:
visited.add(elem_id)
elem = self.nodes[elem_id]
if not value_in_path_get(elem, NodePath.is_phantom,
False):
extracted = value_in_path(elem, path)
if isinstance(extracted, str):
result[extracted] = result.get(extracted,
0) + 1
# check if there is already a successor summary: stop the traversal and take the result.
existing = value_in_path(elem,
NodePath.descendant_summary)
if existing and isinstance(existing, dict):
for summary_item, count in existing.items():
result[summary_item] = result.get(
summary_item, 0) + count
else:
visit_next.extend(
a for a in self.successors(elem_id, edge_type)
if a not in visited)
to_visit = visit_next
return result
def test_value_in_path() -> None:
js = {"foo": {"bla": {"test": 123}}}
assert value_in_path(js, ["foo", "bla", "test"]) == 123
assert value_in_path_get(
js, ["foo", "bla", "test"],
"foo") == "foo" # expected string got int -> default value
assert value_in_path(js, ["foo", "bla", "test", "bar"]) is None
assert value_in_path_get(js, ["foo", "bla", "test", "bar"], 123) == 123
assert value_in_path(js, ["foo", "bla", "bar"]) is None
assert value_in_path_get(js, ["foo", "bla", "bar"], "foo") == "foo"
def expect_expires(reported: Json,
expires: Optional[Union[str, Pattern[Any]]]) -> None:
result = adjuster.adjust({"reported": reported})
if expires is None:
assert value_in_path(result, ["metadata", "expires"]) is None
elif isinstance(expires, str):
assert value_in_path(result, ["metadata", "expires"]) == expires
elif isinstance(expires, Pattern):
assert expires.fullmatch(
value_in_path(result, ["metadata", "expires"]))
def adjust(self, json: Json) -> Json:
def first_matching(paths: List[List[str]]) -> Optional[str]:
for path in paths:
value: Optional[str] = value_in_path(json, path)
if value:
return value
return None
try:
expires_tag = first_matching(self.expires_value)
expires: Optional[str] = None
if expires_tag:
expires = DateTimeKind.from_datetime(expires_tag)
else:
expiration_tag = first_matching(self.expiration_values)
if expiration_tag and expires_tag != "never" and DurationRe.fullmatch(
expiration_tag):
ctime_str = value_in_path(json, NodePath.reported_ctime)
if ctime_str:
ctime = from_utc(ctime_str)
expires = DateTimeKind.from_duration(
expiration_tag, ctime)
if expires:
if "metadata" not in json:
json["metadata"] = {}
json["metadata"]["expires"] = expires
except Exception as ex:
log.debug(f"Could not parse expiration: {ex}")
# json is mutated to save memory
return json
def validate_config_entry(task_data: Json) -> Optional[Json]:
def validate_core_config() -> Optional[Json]:
config = value_in_path(task_data, ["config", ResotoCoreRoot])
if isinstance(config, dict):
# try to read editable config, throws if there are errors
read = from_js(config, EditableConfig)
return read.validate()
else:
return {"error": "Expected a json object"}
def validate_commands_config() -> Optional[Json]:
config = value_in_path(task_data,
["config", ResotoCoreCommandsRoot])
if isinstance(config, dict):
# try to read editable config, throws if there are errors
read = from_js(config, CustomCommandsConfig)
return read.validate()
else:
return {"error": "Expected a json object"}
holder = value_in_path(task_data, ["config"])
if not isinstance(holder, dict):
return {"error": "Expected a json object in config"}
elif ResotoCoreRoot in holder:
return validate_core_config()
elif ResotoCoreCommandsRoot in holder:
return validate_commands_config()
else:
return {"error": "No known configuration found"}
def check_complete(self) -> None:
# check that all vertices are given, that were defined in any edge definition
# note: DiGraph will create an empty vertex node automatically
for node_id, node in self.graph.nodes(data=True):
assert node.get(Section.reported), f"{node_id} was used in an edge definition but not provided as vertex!"
edge_types = {edge[2] for edge in self.graph.edges(data="edge_type")}
al = EdgeType.all
assert not edge_types.difference(al), f"Graph contains unknown edge types! Given: {edge_types}. Known: {al}"
# make sure there is only one root node
rid = GraphAccess.root_id(self.graph)
root_node = self.graph.nodes[rid]
# make sure the root
if value_in_path(root_node, NodePath.reported_kind) == "graph_root" and rid != "root":
# remove node with wrong id +
root_node = self.graph.nodes[rid]
root_node["id"] = "root"
self.graph.add_node("root", **root_node)
for succ in list(self.graph.successors(rid)):
for edge_type in EdgeType.all:
key = GraphAccess.edge_key(rid, succ, edge_type)
if self.graph.has_edge(rid, succ, key):
self.graph.remove_edge(rid, succ, key)
self.add_edge("root", succ, edge_type)
self.graph.remove_node(rid)
def validate_core_config() -> Optional[Json]:
config = value_in_path(task_data, ["config", ResotoCoreRoot])
if isinstance(config, dict):
# try to read editable config, throws if there are errors
read = from_js(config, EditableConfig)
return read.validate()
else:
return {"error": "Expected a json object"}
def render_dot(gen: Iterator[JsonElement]) -> Generator[str, None, None]:
# We use the paired12 color scheme: https://graphviz.org/doc/info/colors.html with color names as 1-12
cit = count_iterator()
icon_map = generate_icon_map()
colors: Dict[str, int] = defaultdict(lambda: (next(cit) % 12) + 1)
node = "node [shape=plain colorscheme=paired12]"
edge = "edge [arrowsize=0.5]"
yield render_dot_header(node, edge)
in_account: Dict[str, List[str]] = defaultdict(list)
for item in gen:
if isinstance(item, dict):
type_name = item.get("type")
if type_name == "node":
uid = value_in_path(item, NodePath.node_id)
if uid:
name = value_in_path_get(item, NodePath.reported_name,
"n/a")
kind = value_in_path_get(item, NodePath.reported_kind,
"n/a")
account = value_in_path_get(item,
NodePath.ancestor_account_name,
"graph_root")
id = value_in_path_get(item, NodePath.reported_id, "n/a")
parsed_kind = parse_kind(kind)
paired12 = kind_colors.get(parsed_kind, colors[kind])
in_account[account].append(uid)
resource = ResourceDescription(uid, name, id, parsed_kind,
kind)
yield render_resource(resource, icon_map, paired12)
elif type_name == "edge":
from_node = value_in_path(item, NodePath.from_node)
to_node = value_in_path(item, NodePath.to_node)
if from_node and to_node:
yield f' "{from_node}" -> "{to_node}" '
else:
raise AttributeError(
f"Expect json object but got: {type(item)}: {item}")
# All elements in the same account are rendered as dedicated subgraph
for account, uids in in_account.items():
yield f' subgraph "{account}" {{'
for uid in uids:
yield f' "{uid}"'
yield " }"
yield "}"
async def respond_dot(
gen: AsyncIterator[JsonElement]) -> AsyncGenerator[str, None]:
# We use the paired12 color scheme: https://graphviz.org/doc/info/colors.html with color names as 1-12
cit = count_iterator()
colors: Dict[str, int] = defaultdict(lambda: (next(cit) % 12) + 1)
node = "node [shape=Mrecord colorscheme=paired12]"
edge = "edge [arrowsize=0.5]"
yield f"digraph {{\nrankdir=LR\noverlap=false\nsplines=true\n{node}\n{edge}"
in_account: Dict[str, List[str]] = defaultdict(list)
async for item in gen:
if isinstance(item, dict):
type_name = item.get("type")
if type_name == "node":
uid = value_in_path(item, NodePath.node_id)
if uid:
name = re.sub(
"[^a-zA-Z0-9]", "",
value_in_path_get(item, NodePath.reported_name, "n/a"))
kind = value_in_path_get(item, NodePath.reported_kind,
"n/a")
account = value_in_path_get(item,
NodePath.ancestor_account_name,
"graph_root")
paired12 = colors[kind]
in_account[account].append(uid)
yield f' "{uid}" [label="{name}|{kind}", style=filled fillcolor={paired12}];'
elif type_name == "edge":
from_node = value_in_path(item, NodePath.from_node)
to_node = value_in_path(item, NodePath.to_node)
edge_type = value_in_path(item, NodePath.edge_type)
if from_node and to_node:
yield f' "{from_node}" -> "{to_node}" [label="{edge_type}"]'
else:
raise AttributeError(
f"Expect json object but got: {type(item)}: {item}")
# All elements in the same account are rendered as dedicated subgraph
for account, uids in in_account.items():
yield f' subgraph "{account}" {{'
for uid in uids:
yield f' "{uid}"'
yield " }"
yield "}"
async def result_to_graph(
gen: AsyncIterator[JsonElement],
render_node: Callable[[Json], Json] = identity) -> DiGraph:
result = DiGraph()
async for item in gen:
if isinstance(item, dict):
type_name = item.get("type")
if type_name == "node":
uid = value_in_path(item, NodePath.node_id)
json_result = render_node(item)
if uid:
result.add_node(uid, **json_result)
elif type_name == "edge":
from_node = value_in_path(item, NodePath.from_node)
to_node = value_in_path(item, NodePath.to_node)
if from_node and to_node:
result.add_edge(from_node, to_node)
else:
raise AttributeError(
f"Expect json object but got: {type(item)}: {item}")
return result
async def do_work(worker_id: str,
task_descriptions: List[WorkerTaskDescription]) -> None:
async with task_queue.attach(worker_id, task_descriptions) as tasks:
while True:
task: WorkerTask = await tasks.get()
incoming_tasks.append(task)
performed_by[task.id].append(worker_id)
if task.name == success.name:
await task_queue.acknowledge_task(worker_id, task.id,
{"result": "done!"})
elif task.name == fail.name:
await task_queue.error_task(worker_id, task.id, ";)")
elif task.name == wait.name:
# if we come here, neither success nor failure was given, ignore the task
pass
elif task.name == WorkerTaskName.validate_config:
cfg_id = task.attrs["config_id"]
if cfg_id == "invalid_config":
await task_queue.error_task(worker_id, task.id,
"Invalid Config ;)")
else:
await task_queue.acknowledge_task(
worker_id, task.id, None)
elif task.name == WorkerTaskName.tag:
node = task.data["node"]
for key in GraphResolver.resolved_ancestors.keys():
for section in Section.content:
if section in node:
node[section].pop(key, None)
# update or delete tags
if "tags" not in node:
node["tags"] = {}
if task.data.get("delete"):
for a in task.data.get("delete"): # type: ignore
node["tags"].pop(a, None)
elif task.data.get("update"):
for k, v in task.data.get(
"update").items(): # type: ignore
node["tags"][k] = v
# for testing purposes: change revision number
kind: str = value_in_path(
node, NodePath.reported_kind) # type: ignore
if kind == "bla":
node["revision"] = "changed"
await task_queue.acknowledge_task(worker_id, task.id, node)
def ancestor_of(self, node_id: str, edge_type: str, kind: str) -> Optional[Json]:
# note: we are using breadth first search here on purpose.
# if there is an ancestor with less distance to this node, we should use this one
next_level = [node_id]
while next_level:
parents: List[str] = []
for p_id in next_level:
p: Json = self.nodes[p_id]
kinds: Optional[List[str]] = value_in_path(p, NodePath.kinds)
if kinds and kind in kinds:
return p
else:
parents.extend(self.predecessors(p_id, edge_type))
next_level = parents
return None
def test_migration() -> None:
cfg1 = migrate_config(
dict(resotocore=dict(runtime=dict(analytics_opt_out=True))))
assert value_in_path(cfg1, "resotocore.runtime.usage_metrics") is False
assert value_in_path(cfg1, "resotocore.runtime.analytics_opt_out") is None
cfg2 = migrate_config(
dict(resotocore=dict(runtime=dict(usage_metrics=True))))
assert value_in_path(cfg2, "resotocore.runtime.usage_metrics") is True
assert value_in_path(cfg1, "resotocore.runtime.analytics_opt_out") is None
cfg3 = migrate_config(
dict(resotocore=dict(
runtime=dict(analytics_opt_out=True, usage_metrics=True))))
assert value_in_path(cfg3, "resotocore.runtime.usage_metrics") is True
assert value_in_path(cfg1, "resotocore.runtime.analytics_opt_out") is None
def expect(jsons: List[Json], path: List[str], value: JsonElement) -> None:
for js in jsons:
v = value_in_path(js, path)
assert v is not None
assert v == value
def from_node() -> Generator[Json, Any, None]:
for node in node_list:
yield node
node_ids = [value_in_path(a, NodePath.node_id) for a in node_list]
for from_n, to_n in interleave(node_ids):
yield {"type": "edge", "from": from_n, "to": to_n}
def expect_expires(reported: Json, expires: Optional[str]) -> None:
result = adjuster.adjust({"reported": reported})
assert value_in_path(result, ["metadata", "expires"]) == expires
def first_matching(paths: List[List[str]]) -> Optional[str]:
for path in paths:
value: Optional[str] = value_in_path(json, path)
if value:
return value
return None
def with_ancestor(ancestor: Json, prop: ResolveProp) -> None:
extracted = value_in_path(ancestor, prop.extract_path)
if extracted:
set_value_in_path(extracted, prop.to_path, node)
