def count_successors_by(node_id: str, edge_type: str, path: List[str]) -> Dict[str, int]: result: Dict[str, int] = {} to_visit = list(self.successors(node_id, edge_type)) while to_visit: visit_next: List[str] = [] for elem_id in to_visit: if elem_id not in visited: visited.add(elem_id) elem = self.nodes[elem_id] if not value_in_path_get(elem, NodePath.is_phantom, False): extracted = value_in_path(elem, path) if isinstance(extracted, str): result[extracted] = result.get(extracted, 0) + 1 # check if there is already a successor summary: stop the traversal and take the result. existing = value_in_path(elem, NodePath.descendant_summary) if existing and isinstance(existing, dict): for summary_item, count in existing.items(): result[summary_item] = result.get( summary_item, 0) + count else: visit_next.extend( a for a in self.successors(elem_id, edge_type) if a not in visited) to_visit = visit_next return result
def test_value_in_path() -> None: js = {"foo": {"bla": {"test": 123}}} assert value_in_path(js, ["foo", "bla", "test"]) == 123 assert value_in_path_get( js, ["foo", "bla", "test"], "foo") == "foo" # expected string got int -> default value assert value_in_path(js, ["foo", "bla", "test", "bar"]) is None assert value_in_path_get(js, ["foo", "bla", "test", "bar"], 123) == 123 assert value_in_path(js, ["foo", "bla", "bar"]) is None assert value_in_path_get(js, ["foo", "bla", "bar"], "foo") == "foo"
def expect_expires(reported: Json, expires: Optional[Union[str, Pattern[Any]]]) -> None: result = adjuster.adjust({"reported": reported}) if expires is None: assert value_in_path(result, ["metadata", "expires"]) is None elif isinstance(expires, str): assert value_in_path(result, ["metadata", "expires"]) == expires elif isinstance(expires, Pattern): assert expires.fullmatch( value_in_path(result, ["metadata", "expires"]))
def adjust(self, json: Json) -> Json: def first_matching(paths: List[List[str]]) -> Optional[str]: for path in paths: value: Optional[str] = value_in_path(json, path) if value: return value return None try: expires_tag = first_matching(self.expires_value) expires: Optional[str] = None if expires_tag: expires = DateTimeKind.from_datetime(expires_tag) else: expiration_tag = first_matching(self.expiration_values) if expiration_tag and expires_tag != "never" and DurationRe.fullmatch( expiration_tag): ctime_str = value_in_path(json, NodePath.reported_ctime) if ctime_str: ctime = from_utc(ctime_str) expires = DateTimeKind.from_duration( expiration_tag, ctime) if expires: if "metadata" not in json: json["metadata"] = {} json["metadata"]["expires"] = expires except Exception as ex: log.debug(f"Could not parse expiration: {ex}") # json is mutated to save memory return json
def validate_config_entry(task_data: Json) -> Optional[Json]: def validate_core_config() -> Optional[Json]: config = value_in_path(task_data, ["config", ResotoCoreRoot]) if isinstance(config, dict): # try to read editable config, throws if there are errors read = from_js(config, EditableConfig) return read.validate() else: return {"error": "Expected a json object"} def validate_commands_config() -> Optional[Json]: config = value_in_path(task_data, ["config", ResotoCoreCommandsRoot]) if isinstance(config, dict): # try to read editable config, throws if there are errors read = from_js(config, CustomCommandsConfig) return read.validate() else: return {"error": "Expected a json object"} holder = value_in_path(task_data, ["config"]) if not isinstance(holder, dict): return {"error": "Expected a json object in config"} elif ResotoCoreRoot in holder: return validate_core_config() elif ResotoCoreCommandsRoot in holder: return validate_commands_config() else: return {"error": "No known configuration found"}
def check_complete(self) -> None: # check that all vertices are given, that were defined in any edge definition # note: DiGraph will create an empty vertex node automatically for node_id, node in self.graph.nodes(data=True): assert node.get(Section.reported), f"{node_id} was used in an edge definition but not provided as vertex!" edge_types = {edge[2] for edge in self.graph.edges(data="edge_type")} al = EdgeType.all assert not edge_types.difference(al), f"Graph contains unknown edge types! Given: {edge_types}. Known: {al}" # make sure there is only one root node rid = GraphAccess.root_id(self.graph) root_node = self.graph.nodes[rid] # make sure the root if value_in_path(root_node, NodePath.reported_kind) == "graph_root" and rid != "root": # remove node with wrong id + root_node = self.graph.nodes[rid] root_node["id"] = "root" self.graph.add_node("root", **root_node) for succ in list(self.graph.successors(rid)): for edge_type in EdgeType.all: key = GraphAccess.edge_key(rid, succ, edge_type) if self.graph.has_edge(rid, succ, key): self.graph.remove_edge(rid, succ, key) self.add_edge("root", succ, edge_type) self.graph.remove_node(rid)
def validate_core_config() -> Optional[Json]: config = value_in_path(task_data, ["config", ResotoCoreRoot]) if isinstance(config, dict): # try to read editable config, throws if there are errors read = from_js(config, EditableConfig) return read.validate() else: return {"error": "Expected a json object"}
def render_dot(gen: Iterator[JsonElement]) -> Generator[str, None, None]: # We use the paired12 color scheme: https://graphviz.org/doc/info/colors.html with color names as 1-12 cit = count_iterator() icon_map = generate_icon_map() colors: Dict[str, int] = defaultdict(lambda: (next(cit) % 12) + 1) node = "node [shape=plain colorscheme=paired12]" edge = "edge [arrowsize=0.5]" yield render_dot_header(node, edge) in_account: Dict[str, List[str]] = defaultdict(list) for item in gen: if isinstance(item, dict): type_name = item.get("type") if type_name == "node": uid = value_in_path(item, NodePath.node_id) if uid: name = value_in_path_get(item, NodePath.reported_name, "n/a") kind = value_in_path_get(item, NodePath.reported_kind, "n/a") account = value_in_path_get(item, NodePath.ancestor_account_name, "graph_root") id = value_in_path_get(item, NodePath.reported_id, "n/a") parsed_kind = parse_kind(kind) paired12 = kind_colors.get(parsed_kind, colors[kind]) in_account[account].append(uid) resource = ResourceDescription(uid, name, id, parsed_kind, kind) yield render_resource(resource, icon_map, paired12) elif type_name == "edge": from_node = value_in_path(item, NodePath.from_node) to_node = value_in_path(item, NodePath.to_node) if from_node and to_node: yield f' "{from_node}" -> "{to_node}" ' else: raise AttributeError( f"Expect json object but got: {type(item)}: {item}") # All elements in the same account are rendered as dedicated subgraph for account, uids in in_account.items(): yield f' subgraph "{account}" {{' for uid in uids: yield f' "{uid}"' yield " }" yield "}"
async def respond_dot( gen: AsyncIterator[JsonElement]) -> AsyncGenerator[str, None]: # We use the paired12 color scheme: https://graphviz.org/doc/info/colors.html with color names as 1-12 cit = count_iterator() colors: Dict[str, int] = defaultdict(lambda: (next(cit) % 12) + 1) node = "node [shape=Mrecord colorscheme=paired12]" edge = "edge [arrowsize=0.5]" yield f"digraph {{\nrankdir=LR\noverlap=false\nsplines=true\n{node}\n{edge}" in_account: Dict[str, List[str]] = defaultdict(list) async for item in gen: if isinstance(item, dict): type_name = item.get("type") if type_name == "node": uid = value_in_path(item, NodePath.node_id) if uid: name = re.sub( "[^a-zA-Z0-9]", "", value_in_path_get(item, NodePath.reported_name, "n/a")) kind = value_in_path_get(item, NodePath.reported_kind, "n/a") account = value_in_path_get(item, NodePath.ancestor_account_name, "graph_root") paired12 = colors[kind] in_account[account].append(uid) yield f' "{uid}" [label="{name}|{kind}", style=filled fillcolor={paired12}];' elif type_name == "edge": from_node = value_in_path(item, NodePath.from_node) to_node = value_in_path(item, NodePath.to_node) edge_type = value_in_path(item, NodePath.edge_type) if from_node and to_node: yield f' "{from_node}" -> "{to_node}" [label="{edge_type}"]' else: raise AttributeError( f"Expect json object but got: {type(item)}: {item}") # All elements in the same account are rendered as dedicated subgraph for account, uids in in_account.items(): yield f' subgraph "{account}" {{' for uid in uids: yield f' "{uid}"' yield " }" yield "}"
async def result_to_graph( gen: AsyncIterator[JsonElement], render_node: Callable[[Json], Json] = identity) -> DiGraph: result = DiGraph() async for item in gen: if isinstance(item, dict): type_name = item.get("type") if type_name == "node": uid = value_in_path(item, NodePath.node_id) json_result = render_node(item) if uid: result.add_node(uid, **json_result) elif type_name == "edge": from_node = value_in_path(item, NodePath.from_node) to_node = value_in_path(item, NodePath.to_node) if from_node and to_node: result.add_edge(from_node, to_node) else: raise AttributeError( f"Expect json object but got: {type(item)}: {item}") return result
async def do_work(worker_id: str, task_descriptions: List[WorkerTaskDescription]) -> None: async with task_queue.attach(worker_id, task_descriptions) as tasks: while True: task: WorkerTask = await tasks.get() incoming_tasks.append(task) performed_by[task.id].append(worker_id) if task.name == success.name: await task_queue.acknowledge_task(worker_id, task.id, {"result": "done!"}) elif task.name == fail.name: await task_queue.error_task(worker_id, task.id, ";)") elif task.name == wait.name: # if we come here, neither success nor failure was given, ignore the task pass elif task.name == WorkerTaskName.validate_config: cfg_id = task.attrs["config_id"] if cfg_id == "invalid_config": await task_queue.error_task(worker_id, task.id, "Invalid Config ;)") else: await task_queue.acknowledge_task( worker_id, task.id, None) elif task.name == WorkerTaskName.tag: node = task.data["node"] for key in GraphResolver.resolved_ancestors.keys(): for section in Section.content: if section in node: node[section].pop(key, None) # update or delete tags if "tags" not in node: node["tags"] = {} if task.data.get("delete"): for a in task.data.get("delete"): # type: ignore node["tags"].pop(a, None) elif task.data.get("update"): for k, v in task.data.get( "update").items(): # type: ignore node["tags"][k] = v # for testing purposes: change revision number kind: str = value_in_path( node, NodePath.reported_kind) # type: ignore if kind == "bla": node["revision"] = "changed" await task_queue.acknowledge_task(worker_id, task.id, node)
def ancestor_of(self, node_id: str, edge_type: str, kind: str) -> Optional[Json]: # note: we are using breadth first search here on purpose. # if there is an ancestor with less distance to this node, we should use this one next_level = [node_id] while next_level: parents: List[str] = [] for p_id in next_level: p: Json = self.nodes[p_id] kinds: Optional[List[str]] = value_in_path(p, NodePath.kinds) if kinds and kind in kinds: return p else: parents.extend(self.predecessors(p_id, edge_type)) next_level = parents return None
def test_migration() -> None: cfg1 = migrate_config( dict(resotocore=dict(runtime=dict(analytics_opt_out=True)))) assert value_in_path(cfg1, "resotocore.runtime.usage_metrics") is False assert value_in_path(cfg1, "resotocore.runtime.analytics_opt_out") is None cfg2 = migrate_config( dict(resotocore=dict(runtime=dict(usage_metrics=True)))) assert value_in_path(cfg2, "resotocore.runtime.usage_metrics") is True assert value_in_path(cfg1, "resotocore.runtime.analytics_opt_out") is None cfg3 = migrate_config( dict(resotocore=dict( runtime=dict(analytics_opt_out=True, usage_metrics=True)))) assert value_in_path(cfg3, "resotocore.runtime.usage_metrics") is True assert value_in_path(cfg1, "resotocore.runtime.analytics_opt_out") is None
def expect(jsons: List[Json], path: List[str], value: JsonElement) -> None: for js in jsons: v = value_in_path(js, path) assert v is not None assert v == value
def from_node() -> Generator[Json, Any, None]: for node in node_list: yield node node_ids = [value_in_path(a, NodePath.node_id) for a in node_list] for from_n, to_n in interleave(node_ids): yield {"type": "edge", "from": from_n, "to": to_n}
def expect_expires(reported: Json, expires: Optional[str]) -> None: result = adjuster.adjust({"reported": reported}) assert value_in_path(result, ["metadata", "expires"]) == expires
def first_matching(paths: List[List[str]]) -> Optional[str]: for path in paths: value: Optional[str] = value_in_path(json, path) if value: return value return None
def with_ancestor(ancestor: Json, prop: ResolveProp) -> None: extracted = value_in_path(ancestor, prop.extract_path) if extracted: set_value_in_path(extracted, prop.to_path, node)