def migrate_for_subfile(u, *, scope): for resolver in u.resolvers: uu = u.new_child(resolver) if uu.has("paths", resolver=resolver): from dictknife import DictWalker, Or method_walker = DictWalker( [Or(["get", "post", "put", "delete", "patch", "head"])] ) schema_walker = DictWalker(["schema"]) for path, sd in method_walker.walk(uu.resolver.doc["paths"]): # parameters # responses frame = {} if "produces" in sd[path[-1]]: ref = path_to_json_pointer(["paths", *path, "produces"]) frame["produces"] = uu.pop(ref) with scope.scope(frame or None): if "responses" in sd[path[-1]]: for spath, ssd in schema_walker.walk(sd[path[-1]]["responses"]): fullpath = ["paths", *path, "responses", *spath] ref = path_to_json_pointer(fullpath) schema = uu.pop(ref) content = uu.make_dict() for produce in scope[["produces"]]: content[produce] = {"schema": schema} ref = path_to_json_pointer([*fullpath[:-1], "content"]) uu.update(ref, content)
def __init__(self, resolver, *, store: _yaml.NodeStore): self.resolver = resolver self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker([is_ref]) self.errors = [] self.store = store
def run(filename: str) -> str: d = loading.loadfile(filename) w = DictWalker(["vars", "exec"]) for path, w in w.walk(d): # path: services/<service_name>/vars/exec service_name = path[-3] exec_path = w["exec"] if exec_path == "./enduser_server": continue if exec_path == "./tuner_server": continue if _normalize(_normalize2(exec_path)).startswith( _normalize(_normalize2(service_name)) ): continue loading.dumpfile( { "must_include_part": _normalize(_normalize2(service_name)), "ref": f"#/{'/'.join(path[:-2])}", "exec_path": exec_path, }, format="json", ) print("")
def __init__(self, resolver, *, store: Store): self.resolver = resolver self.store = store self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker(["$ref"]) self.errors = []
class Expander: def __init__(self, resolver): self.resolver = resolver self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker(["$ref"]) self.errors = [] def expand(self, doc=None, resolver=None, ctx=None): doc = doc or self.resolver.doc resolver = resolver or self.resolver if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc = self.expand(original, resolver=self.accessor.resolver, ctx=ctx) self.accessor.pop_stack() return new_doc else: for path, sd in self.ref_walking.iterate(doc): try: new_sd = self.expand(sd, resolver=resolver, ctx=ctx) container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except Exception as e: self.errors.append(ReferenceError(e, path=path[:], data=sd)) return doc
def run(filename: str, output: t.Optional[str] = None): d = loading.loadfile(filename) for path, sd in DictWalker(["allOf"]).walk(d): parent = d for name in path[:-2]: parent = parent[name] assert parent[path[-2]] == sd parent[path[-2]] = sd.pop("allOf")[0] loading.dumpfile(d, output)
def build_subset(resolver, ref): subset = {} ob = resolver.access_by_json_pointer(ref) resolver.assign_by_json_pointer(ref, ob, doc=subset) for path, sd in DictWalker(["$ref"]).walk(ob): # xxx: if sd["$ref"].startswith("#/"): resolver.assign(path[:-1], sd, doc=subset) return subset
class Scaner: def __init__(self, resolver, *, store: Store): self.resolver = resolver self.store = store self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker([is_ref]) self.errors = [] def scan(self, doc=None, resolver=None): if not doc and doc is not None: return doc resolver = resolver or self.resolver try: doc = doc or resolver.doc except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append(ParseError(e, store=self.store)) if doc is None: doc = {} doc, _ = self._scan(doc, resolver=resolver, seen={}) return doc def _scan(self, doc, *, resolver, seen: dict): if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc, _ = self._scan( original, resolver=self.accessor.resolver, seen=seen ) return new_doc, self.accessor.pop_stack() else: for path, sd in self.ref_walking.iterate(doc): try: uid = id(sd) if uid in seen: continue seen[uid] = sd new_sd, sresolver = self._scan(sd, resolver=resolver, seen=seen) if resolver.filename != sresolver.filename: container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except (KeyError, FileNotFoundError) as e: self.errors.append( ReferenceError(e, store=self.store, path=path[:], data=sd) ) except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append( ParseError(e, store=self.store, path=path[:], data=sd) ) return doc, resolver
def transform(self, rawtarget, depth): d = self.oas2transformer.transform(rawtarget, depth) for _, sd in DictWalker(["$ref"]).walk(d): sd["$ref"] = sd["$ref"].replace("#/definitions/", "#/components/schemas/") if "components" not in d: d["components"] = {} if "schemas" not in d["components"]: d["components"]["schemas"] = {} d["components"]["schemas"] = d.pop("definitions", {}) return d
def main(*, src: str) -> None: precompile_ref_walker = DictWalker(["$precompile-ref"]) def onload(d, resolver): for path, sd in precompile_ref_walker.walk(d): subresolver, query = resolver.resolve(sd.pop("$precompile-ref")) value = subresolver.access(subresolver.doc, query) jsref = path_to_json_pointer(path[:-1]) resolver.assign(d, jsref, value) d = bundle(src, onload=onload) loading.dumpfile(d)
def migrate_for_subfile(u, *, scope): for resolver in u.resolvers: uu = u.new_child(resolver) if uu.has("paths", resolver=resolver): from dictknife import DictWalker, Or method_walker = DictWalker( [Or(["get", "post", "put", "delete", "patch", "head"])]) schema_walker = DictWalker(["schema"]) for path, sd in method_walker.walk(uu.resolver.doc["paths"]): # parameters # responses frame = {} if "produces" in sd[path[-1]]: ref = path_to_json_pointer(["paths", *path, "produces"]) frame["produces"] = uu.pop(ref) with scope.scope(frame or None): if "responses" in sd[path[-1]]: for spath, ssd in schema_walker.walk( sd[path[-1]]["responses"]): fullpath = ["paths", *path, "responses", *spath] ref = path_to_json_pointer(fullpath) schema = uu.pop(ref) content = uu.make_dict() for produce in scope[["produces"]]: content[produce] = {"schema": schema} ref = path_to_json_pointer( [*fullpath[:-1], "content"]) uu.update(ref, content)
def main(*, src: str) -> None: precompile_ref_walker = DictWalker(["$precompile-ref"]) accessor = Accessor() def onload(d, subresolver): for path, sd in precompile_ref_walker.walk(d): sdoc, query = subresolver.resolve(sd.pop("$precompile-ref")) sresolved = access_by_json_pointer(sdoc.doc, query) accessor.assign(d, path[:-1], sresolved) resolver = get_resolver_from_filename(src, onload=onload) d = Bundler(resolver).bundle() loading.dumpfile(d)
class Scaner: def __init__(self, resolver, *, store: Store): self.resolver = resolver self.store = store self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker(["$ref"]) self.errors = [] def scan(self, doc=None, resolver=None, ctx=None): resolver = resolver or self.resolver try: doc = doc or resolver.doc except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append(ParseError(e, store=self.store)) if doc is None: doc = {} if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc = self.scan(original, resolver=self.accessor.resolver, ctx=ctx) self.accessor.pop_stack() return new_doc else: for path, sd in self.ref_walking.iterate(doc): try: new_sd = self.scan(sd, resolver=resolver, ctx=ctx) container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except (KeyError, FileNotFoundError) as e: self.errors.append( ReferenceError(e, store=self.store, path=path[:], data=sd)) except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append( ParseError(e, store=self.store, path=path[:], data=sd)) return doc
def migrate_refs(uu, data, *, scope, walker=DictWalker(["$ref"])): for path, d in walker.walk(data): if "/definitions/" in d["$ref"]: uu.update_by_path( path, d["$ref"].replace("/definitions/", "/components/schemas/", 1)) if "/parameters/" in d["$ref"]: uu.update_by_path( path, d["$ref"].replace("/parameters/", "/components/parameters/", 1)) if "/responses/" in d["$ref"]: uu.update_by_path( path, d["$ref"].replace("/responses/", "/components/responses/", 1))
def merge(*, extracted, jsonschema): schemas = {} paths = extracted["paths"] for name, s in jsonschema["definitions"].items(): if "$ref" in s: s["$ref"].replace("#definitions", "#/components/schemas") schemas[name] = s else: schemas[name] = s # remove needless title property for path, sd in DictWalker(["title"]).walk(schemas): if path[-2] == sd["title"]: sd.pop("title") return {"components": {"schemas": schemas}, "paths": paths}
def json2swagger( *, files, dst: str, output_format: str, name: str, detector, emitter, annotate, emit, with_minimap: bool, without_example: bool ): from prestring import Module from dictknife import DictWalker if annotate is not None: annotate = loading.loadfile(annotate) else: annotate = {} ns = "dictknife.swaggerknife.json2swagger" detector = import_symbol(detector, ns=ns)() emitter = import_symbol(emitter, ns=ns)(annotate) info = None for src in files: data = loading.loadfile(src) info = detector.detect(data, name, info=info) if emit == "info": loading.dumpfile(info, filename=dst) else: m = Module(indent=" ") m.stmt(name) emitter.emit(info, m) if with_minimap: print("# minimap ###") print("# *", end="") print("\n# ".join(str(m).split("\n"))) if without_example: for _, d in DictWalker(["example"]).walk(emitter.doc): d.pop("example") loading.dumpfile(emitter.doc, filename=dst, format=output_format)
from prestring.python import Module from dictknife import DictWalker from dictknife import loading w = DictWalker(["lines"]) d = loading.loadfile(format="json") r = [] for _, d in w.walk(d): if d["language"] == "python" or d["language"] == "py": r.append(d["lines"]) m = Module() m.from_("nbreversible", "code") for lines in r: with m.with_("code()"): for line in lines: if line.startswith("%"): m.stmt("#{}", line) else: m.stmt(line.rstrip()) m.sep() print(m)
def onload(d, resolver, w=DictWalker(["$include"])): for _, sd, in w.walk(d): subresolver, jsref = resolver.resolve(sd.pop("$include")) sd.update(subresolver.access_by_json_pointer(jsref))
def __init__(self, resolver): self.resolver = resolver self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker(["$ref"])
def ref_walking(self): return DictWalker([is_ref])
import pathlib import importlib.util from dictknife import loading from dictknife import Accessor from dictknife import DictWalker spec = importlib.util.find_spec("botocore") path = pathlib.Path(spec.origin) if path.name == "__init__.py": path = path.parent d = loading.loadfile(path / ("data/sqs/2012-11-05/service-2.json")) dst = {} a = Accessor(make_dict=dict) for name, sd in d["operations"].items(): path = ["operations", name] a.assign(dst, path, sd) ssd = a.access(d, ["shapes", sd["input"]["shape"]]) a.assign(dst, ["shapes", sd["input"]["shape"]], ssd) if "output" in sd: ssd = a.access(d, ["shapes", sd["output"]["shape"]]) a.assign(dst, ["shapes", sd["output"]["shape"]], ssd) # slim-up for path, sd in DictWalker(["documentation"]).walk(dst): sd.pop("documentation") loading.dumpfile(dst, format="json")
def ref_walker(self): return DictWalker([And(["$ref", _is_string])])
def ref_walking(self): return DictWalker(["$ref"])
def migrate_for_subfile(uu, *, scope, callbacks, ref_wawlker=DictWalker(["$ref"])): migrate_refs(uu, uu.resolver.doc, scope=scope) if uu.has("definitions"): uu.update_by_path(["components", "schemas"], uu.pop_by_path(["definitions"])) if uu.has("parameters"): request_bodies = migrate_parameters(uu, uu.resolver.doc, path=["parameters"], scope=scope) uu.update_by_path(["components", "parameters"], uu.pop_by_path(["parameters"])) # for in:body if request_bodies: names = list(request_bodies.keys()) # todo: optimizatin? def fixref(uu, *, names=names): will_be_remove_paths_if_empty = set() for path, sd in ref_wawlker.walk(uu.resolver.doc): ref = sd["$ref"] if is_empty(ref): continue if "#/components/parameters" not in ref: continue if not any(name in ref for name in names): continue uu.pop_by_path(path[:-1]) new_value = { "$ref": ref.replace("#/components/parameters", "#/components/requestBodies") } if path[0] == "paths": new_path = itertools.takewhile( lambda x: x != "parameters", path) uu.update_by_path([*new_path, "requestBody"], new_value) elif path[0] == "components": # #/components/parameters/<name> uu.update_by_path( ["components", "requestBodies", path[2]], new_value) else: raise RuntimeError("unexpected path: {}".format(path)) will_be_remove_paths_if_empty.add(tuple(path[:-2])) for path in will_be_remove_paths_if_empty: if is_empty_collection(uu.resolver.access(path)): uu.pop_by_path(path) callbacks.append(fixref) for name, body in request_bodies.items(): uu.update_by_path(["components", "requestBodies", name], body) if uu.has("responses"): uu.update_by_path(["components", "responses"], uu.pop_by_path(["responses"])) if uu.has("securityDefinitions"): uu.update_by_path(["components", "securitySchemes"], uu.pop_by_path(["securityDefinitions"])) # todo: requestBodies frame = make_dict() frame.update( migrate_parameters(uu, uu.resolver.doc, path=["parameters"], scope=scope)) with scope.scope(frame or None): if uu.has("paths"): migrate_paths(uu, uu.resolver.doc, scope=scope)
def migrate_paths(uu, data, *, scope, schema_walker=DictWalker(["schema"])): for url_path, path_item in data["paths"].items(): # xxx: vendor extensions? if url_path.startswith("x-"): continue # todo: parse pathItem object # https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md#path-item-object operation_methods = [ "get", "put", "post", "delete", "options", "head", "patch" ] frame = make_dict() frame.update( migrate_parameters(uu, path_item, path=["paths", url_path, "parameters"], scope=scope)) with scope.scope(frame or None): for method_name in operation_methods: operation = path_item.get(method_name) if operation is None: continue # todo: parse Operation object # https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md#operation-object frame = make_dict() # parameters frame.update( migrate_parameters( uu, operation, path=["paths", url_path, method_name, "parameters"], scope=scope, )) # produces if "produces" in operation: frame["produces"] = uu.pop_by_path( ["paths", url_path, method_name, "produces"]) # consumes if "consumes" in operation: frame["consumes"] = uu.pop_by_path( ["paths", url_path, method_name, "consumes"]) # responses with scope.scope(frame or None): # requestBody request_body = scope.get(["requestBody"]) if request_body is not None: uu.update_by_path( ["paths", url_path, method_name, "requestBody"], request_body, ) if "responses" in operation: for spath, sd in schema_walker.walk( operation["responses"]): fullpath = [ "paths", url_path, method_name, "responses", *spath, ] schema = uu.pop_by_path(fullpath) content = uu.make_dict() for produce in scope[["produces"]]: content[produce] = {"schema": schema} uu.update_by_path([*fullpath[:-1], "content"], content)
from dictknife import DictWalker # from: https://github.com/BigstickCarpet/json-schema-ref-parser d = loading.loads( """ { "definitions": { "person": { "$ref": "schemas/people/Bruce-Wayne.json" }, "place": { "$ref": "schemas/places.yaml#/definitions/Gotham-City" }, "thing": { "$ref": "http://wayne-enterprises.com/things/batmobile" }, "color": { "$ref": "#/definitions/thing/properties/colors/black-as-the-night" } } } """, format="json" ) walker = DictWalker(["$ref"]) refs = [("/".join(path[:]), sd["$ref"]) for path, sd in walker.walk(d)] for path, ref in refs: print(path, ref)
def xxx_of_walker(self): return DictWalker([And([Or(_combine_types), _is_array])])
def description_walker(self): return DictWalker(["description"])
def onload(d, resolver, w=DictWalker(["$xref"])): for path, sd in w.walk(d): subresolver, jsonref = resolver.resolve(sd.pop("$xref")) value = subresolver.access_by_json_pointer(jsonref) resolver.assign(path[:-1], value)
class Loader: def __init__(self, resolver, *, store: _yaml.NodeStore): self.resolver = resolver self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker([is_ref]) self.errors = [] self.store = store @property def filename(self) -> str: return self.resolver.filename def load(self, doc=None, resolver=None): if not doc and doc is not None: return doc resolver = resolver or self.resolver try: doc = doc or resolver.doc except _yaml.MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append(ParseError(e, history=[resolver.filename])) if doc is None: doc = {} doc, _ = self._load(doc, resolver=resolver, seen={}) return doc def _load(self, doc, *, resolver, seen: dict): if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc, _ = self._load(original, resolver=self.accessor.resolver, seen=seen) return new_doc, self.accessor.pop_stack() else: for path, sd in self.ref_walking.iterate(doc): try: uid = id(sd) if uid in seen: continue seen[uid] = sd new_sd, sresolver = self._load(sd, resolver=resolver, seen=seen) if resolver.filename != sresolver.filename: container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except FileNotFoundError as e: self.errors.append( ResolutionError( e, path=path[:], data=sd, history=[ r.filename for r in self.accessor.stack[:-1] ], )) except KeyError as e: self.errors.append( ResolutionError( e, path=path[:], data=sd, history=[r.filename for r in self.accessor.stack], )) except _yaml.MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append( ParseError( e, path=path[:], data=sd, history=[ r.filename for r in self.accessor.stack ], )) return doc, resolver
def describe_dict(d, *, life=None, showzero_callable=None): sigmap = {} walker = DictWalker([Or(["$len", "$members"])]) def _describe_type(v): if hasattr(v, "keys"): return dict.__name__ return type(v).__name__ def _show_on_lifezero(d, *, path): if hasattr(d, "keys"): rep = make_dict() for k, v in d.items(): if hasattr(v, "keys"): rep[k] = f"{_describe_type(v)}@{len(v)}" elif isinstance(v, (list, tuple)): rep[k] = f"{_describe_type(v)}@{len(v)}" else: rep[k] = f"{_describe_type(v)}" sig = json.dumps(rep, sort_keys=True, default=str) if sig in sigmap: return sigmap[sig] sigmap[sig] = path_to_json_pointer(path) return rep elif isinstance(d, (list, tuple)): seen = {} for x in d: rep = _show(x, life=0, path=path) sig = json.dumps(rep, sort_keys=True, default=str) if sig in seen: continue seen[sig] = rep return {"$len": len(d), "$cases": list(seen.values())} else: return _describe_type(d) def _show(d, *, path, life, showzero_callable=showzero_callable or _show_on_lifezero): if life == 0: # -1 is full expand return showzero_callable(d, path=path) if hasattr(d, "keys"): rep = make_dict() for k, v in d.items(): path.append(k) if hasattr(v, "__len__") and len(v) == 0: rep[k] = f"{_describe_type(v)}@{len(v)}" elif hasattr(v, "keys"): rep[k] = _show(v, life=life - 1, path=path) elif isinstance(v, (list, tuple)): rep[k] = _show(v, life=life - 1, path=path) else: rep[k] = f"{_describe_type(v)}" path.pop() sig = json.dumps(rep, sort_keys=True, default=str) if sig in sigmap: return sigmap[sig] sigmap[sig] = path_to_json_pointer(path) return rep elif isinstance(d, (list, tuple)): seen = {} members = [] path.append("[]") csigmap = {} for x in d: rep = _show(x, life=life - 1, path=path) sig = json.dumps(rep, sort_keys=True, default=str) if sig in seen: members.append(csigmap[sig]) continue rep = copy.deepcopy(rep) for ks, sd in walker.walk(rep): sd.pop(ks[-1]) seen[sig] = rep csigmap[ sig] = f"{path_to_json_pointer(path)}/$cases/{len(csigmap)}" members.append(csigmap[sig]) rep = {"$len": len(d)} if seen: rep["$cases"] = list(seen.values()) if members: rep["$members"] = members path.pop() sig = json.dumps(rep, sort_keys=True, default=str) if sig in sigmap: return sigmap[sig] sigmap[sig] = path_to_json_pointer(path) return rep else: return _describe_type(d) if life is None: life = int(os.environ.get("LIFE") or "0") rep = _show(d, life=life, path=["#"]) return rep