class Expander: def __init__(self, resolver): self.resolver = resolver self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker(["$ref"]) self.errors = [] def expand(self, doc=None, resolver=None, ctx=None): doc = doc or self.resolver.doc resolver = resolver or self.resolver if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc = self.expand(original, resolver=self.accessor.resolver, ctx=ctx) self.accessor.pop_stack() return new_doc else: for path, sd in self.ref_walking.iterate(doc): try: new_sd = self.expand(sd, resolver=resolver, ctx=ctx) container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except Exception as e: self.errors.append(ReferenceError(e, path=path[:], data=sd)) return doc
def run(*, src: str, schema: str) -> None: with open(schema) as rf: schema = yaml.load(rf) with open(src) as rf: loader = Loader(rf) try: assert loader.check_data() data = loader.get_data() finally: loader.dispose() jsonschema.Draft4Validator.check_schema(schema) validator = jsonschema.Draft4Validator(schema) for err in validator.iter_errors(data): print("E", err) a = Accessor() path = list(err.path) ob = a.access(data, path[:-1]) ev = mem[id(ob)] for kev, vev in ev.value: if kev.value == path[-1]: print("----------------------------------------") print(str(vev.start_mark).lstrip()) lineno = vev.start_mark.line + 1 with open(src) as rf: for i, line in enumerate(rf, 1): if lineno == i: print(f" {i:02d}: -> {line}", end="") else: print(f" {i:02d}: {line}", end="") break
class Emitter: def __init__(self, accessor, item_map): self.raw_accessor = Accessor() self.accessor = accessor self.item_map = item_map @reify def ref_walking(self): return DictWalker([is_ref]) def get_item_by_globalref(self, globalref): return self.accessor.cache[globalref] def get_item_by_localref(self, localref): return self.item_map[localref] def emit(self, resolver, doc, *, conflicted): # side effect d = make_dict() for path, sd in self.ref_walking.iterate(doc): self.replace_ref(resolver, sd) d = deepmerge(d, doc) for name, item in self.item_map.items(): if name == "": continue data = item.data # replace: <file.yaml>#/<ref> -> #/<ref> for path, sd in self.ref_walking.iterate(data): if not sd["$ref"].startswith("#/"): self.replace_ref(item.resolver, sd) if sd["$ref"] in conflicted: self.replace_ref(item.resolver, sd) self.raw_accessor.assign(d, name.split("/"), data) # adhoc paths support will_removes = set() paths = d.get("paths") or {} for path, sub in list(paths.items()): if "$ref" in sub and sub["$ref"].startswith("#/"): related_path = tuple(sub["$ref"][2:].split("/")) paths[path] = self.raw_accessor.access(d, related_path).copy() will_removes.add(related_path) for related_path in will_removes: self.raw_accessor.maybe_remove(d, related_path) return d def replace_ref(self, resolver, sd): filename, _, pointer = resolver.resolve_pathset(sd["$ref"]) related = self.get_item_by_globalref((filename, pointer)) new_ref = "#/{}".format(related.localref) if sd["$ref"] != new_ref: logger.debug( "fix ref: %r -> %r (where=%r)", sd["$ref"], new_ref, resolver.filename ) sd["$ref"] = new_ref
class Scaner: def __init__(self, resolver, *, store: Store): self.resolver = resolver self.store = store self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker([is_ref]) self.errors = [] def scan(self, doc=None, resolver=None): if not doc and doc is not None: return doc resolver = resolver or self.resolver try: doc = doc or resolver.doc except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append(ParseError(e, store=self.store)) if doc is None: doc = {} doc, _ = self._scan(doc, resolver=resolver, seen={}) return doc def _scan(self, doc, *, resolver, seen: dict): if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc, _ = self._scan( original, resolver=self.accessor.resolver, seen=seen ) return new_doc, self.accessor.pop_stack() else: for path, sd in self.ref_walking.iterate(doc): try: uid = id(sd) if uid in seen: continue seen[uid] = sd new_sd, sresolver = self._scan(sd, resolver=resolver, seen=seen) if resolver.filename != sresolver.filename: container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except (KeyError, FileNotFoundError) as e: self.errors.append( ReferenceError(e, store=self.store, path=path[:], data=sd) ) except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append( ParseError(e, store=self.store, path=path[:], data=sd) ) return doc, resolver
class Scaner: def __init__(self, resolver, *, store: Store): self.resolver = resolver self.store = store self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker(["$ref"]) self.errors = [] def scan(self, doc=None, resolver=None, ctx=None): resolver = resolver or self.resolver try: doc = doc or resolver.doc except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append(ParseError(e, store=self.store)) if doc is None: doc = {} if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc = self.scan(original, resolver=self.accessor.resolver, ctx=ctx) self.accessor.pop_stack() return new_doc else: for path, sd in self.ref_walking.iterate(doc): try: new_sd = self.scan(sd, resolver=resolver, ctx=ctx) container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except (KeyError, FileNotFoundError) as e: self.errors.append( ReferenceError(e, store=self.store, path=path[:], data=sd)) except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append( ParseError(e, store=self.store, path=path[:], data=sd)) return doc
import pathlib import importlib.util from dictknife import loading from dictknife import Accessor from dictknife import DictWalker spec = importlib.util.find_spec("botocore") path = pathlib.Path(spec.origin) if path.name == "__init__.py": path = path.parent d = loading.loadfile(path / ("data/sqs/2012-11-05/service-2.json")) dst = {} a = Accessor(make_dict=dict) for name, sd in d["operations"].items(): path = ["operations", name] a.assign(dst, path, sd) ssd = a.access(d, ["shapes", sd["input"]["shape"]]) a.assign(dst, ["shapes", sd["input"]["shape"]], ssd) if "output" in sd: ssd = a.access(d, ["shapes", sd["output"]["shape"]]) a.assign(dst, ["shapes", sd["output"]["shape"]], ssd) # slim-up for path, sd in DictWalker(["documentation"]).walk(dst): sd.pop("documentation") loading.dumpfile(dst, format="json")
class Loader: def __init__(self, resolver, *, store: _yaml.NodeStore): self.resolver = resolver self.accessor = StackedAccessor(resolver) self.accessing = Accessor() self.ref_walking = DictWalker([is_ref]) self.errors = [] self.store = store @property def filename(self) -> str: return self.resolver.filename def load(self, doc=None, resolver=None): if not doc and doc is not None: return doc resolver = resolver or self.resolver try: doc = doc or resolver.doc except _yaml.MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append(ParseError(e, history=[resolver.filename])) if doc is None: doc = {} doc, _ = self._load(doc, resolver=resolver, seen={}) return doc def _load(self, doc, *, resolver, seen: dict): if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc, _ = self._load(original, resolver=self.accessor.resolver, seen=seen) return new_doc, self.accessor.pop_stack() else: for path, sd in self.ref_walking.iterate(doc): try: uid = id(sd) if uid in seen: continue seen[uid] = sd new_sd, sresolver = self._load(sd, resolver=resolver, seen=seen) if resolver.filename != sresolver.filename: container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except FileNotFoundError as e: self.errors.append( ResolutionError( e, path=path[:], data=sd, history=[ r.filename for r in self.accessor.stack[:-1] ], )) except KeyError as e: self.errors.append( ResolutionError( e, path=path[:], data=sd, history=[r.filename for r in self.accessor.stack], )) except _yaml.MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append( ParseError( e, path=path[:], data=sd, history=[ r.filename for r in self.accessor.stack ], )) return doc, resolver
from dictknife import Accessor from dictknife.pp import pp, indent a = Accessor(OrderedDict) d = OrderedDict() # assign a.assign(d, ['a', 'b', 'c'], 'v') with indent(2, 'assign:\n'): print(d) pp(d) print() # access with indent(2, '\naccess: ["a", "b", "c"]\n'): print(['a', 'b', 'c'], a.access(d, ['a', 'b', 'c'])) # print(['a', 'b', 'x'], a.access(d, ['a', 'b', 'x'])) # error # exists with indent(2, '\nexists:\n'): import copy # NOQA d2 = copy.deepcopy(d) print(['a', 'b', 'c'], a.exists(d2, ['a', 'b', 'c'])) print(['a', 'b', 'x'], a.exists(d2, ['a', 'b', 'x'])) # maybe_remove with indent(2, '\nmaybe_remove:\n'): import copy # NOQA d2 = copy.deepcopy(d)