class Expander:
def __init__(self, resolver):
self.resolver = resolver
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker(["$ref"])
self.errors = []
def expand(self, doc=None, resolver=None, ctx=None):
doc = doc or self.resolver.doc
resolver = resolver or self.resolver
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc = self.expand(original,
resolver=self.accessor.resolver,
ctx=ctx)
self.accessor.pop_stack()
return new_doc
else:
for path, sd in self.ref_walking.iterate(doc):
try:
new_sd = self.expand(sd, resolver=resolver, ctx=ctx)
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except Exception as e:
self.errors.append(ReferenceError(e, path=path[:],
data=sd))
return doc
def run(*, src: str, schema: str) -> None:
with open(schema) as rf:
schema = yaml.load(rf)
with open(src) as rf:
loader = Loader(rf)
try:
assert loader.check_data()
data = loader.get_data()
finally:
loader.dispose()
jsonschema.Draft4Validator.check_schema(schema)
validator = jsonschema.Draft4Validator(schema)
for err in validator.iter_errors(data):
print("E", err)
a = Accessor()
path = list(err.path)
ob = a.access(data, path[:-1])
ev = mem[id(ob)]
for kev, vev in ev.value:
if kev.value == path[-1]:
print("----------------------------------------")
print(str(vev.start_mark).lstrip())
lineno = vev.start_mark.line + 1
with open(src) as rf:
for i, line in enumerate(rf, 1):
if lineno == i:
print(f" {i:02d}: -> {line}", end="")
else:
print(f" {i:02d}: {line}", end="")
break
class Emitter:
def __init__(self, accessor, item_map):
self.raw_accessor = Accessor()
self.accessor = accessor
self.item_map = item_map
@reify
def ref_walking(self):
return DictWalker([is_ref])
def get_item_by_globalref(self, globalref):
return self.accessor.cache[globalref]
def get_item_by_localref(self, localref):
return self.item_map[localref]
def emit(self, resolver, doc, *, conflicted):
# side effect
d = make_dict()
for path, sd in self.ref_walking.iterate(doc):
self.replace_ref(resolver, sd)
d = deepmerge(d, doc)
for name, item in self.item_map.items():
if name == "":
continue
data = item.data
# replace: <file.yaml>#/<ref> -> #/<ref>
for path, sd in self.ref_walking.iterate(data):
if not sd["$ref"].startswith("#/"):
self.replace_ref(item.resolver, sd)
if sd["$ref"] in conflicted:
self.replace_ref(item.resolver, sd)
self.raw_accessor.assign(d, name.split("/"), data)
# adhoc paths support
will_removes = set()
paths = d.get("paths") or {}
for path, sub in list(paths.items()):
if "$ref" in sub and sub["$ref"].startswith("#/"):
related_path = tuple(sub["$ref"][2:].split("/"))
paths[path] = self.raw_accessor.access(d, related_path).copy()
will_removes.add(related_path)
for related_path in will_removes:
self.raw_accessor.maybe_remove(d, related_path)
return d
def replace_ref(self, resolver, sd):
filename, _, pointer = resolver.resolve_pathset(sd["$ref"])
related = self.get_item_by_globalref((filename, pointer))
new_ref = "#/{}".format(related.localref)
if sd["$ref"] != new_ref:
logger.debug(
"fix ref: %r -> %r (where=%r)", sd["$ref"], new_ref, resolver.filename
)
sd["$ref"] = new_ref
class Scaner:
def __init__(self, resolver, *, store: Store):
self.resolver = resolver
self.store = store
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker([is_ref])
self.errors = []
def scan(self, doc=None, resolver=None):
if not doc and doc is not None:
return doc
resolver = resolver or self.resolver
try:
doc = doc or resolver.doc
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(ParseError(e, store=self.store))
if doc is None:
doc = {}
doc, _ = self._scan(doc, resolver=resolver, seen={})
return doc
def _scan(self, doc, *, resolver, seen: dict):
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc, _ = self._scan(
original, resolver=self.accessor.resolver, seen=seen
)
return new_doc, self.accessor.pop_stack()
else:
for path, sd in self.ref_walking.iterate(doc):
try:
uid = id(sd)
if uid in seen:
continue
seen[uid] = sd
new_sd, sresolver = self._scan(sd, resolver=resolver, seen=seen)
if resolver.filename != sresolver.filename:
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except (KeyError, FileNotFoundError) as e:
self.errors.append(
ReferenceError(e, store=self.store, path=path[:], data=sd)
)
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(
ParseError(e, store=self.store, path=path[:], data=sd)
)
return doc, resolver
class Scaner:
def __init__(self, resolver, *, store: Store):
self.resolver = resolver
self.store = store
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker(["$ref"])
self.errors = []
def scan(self, doc=None, resolver=None, ctx=None):
resolver = resolver or self.resolver
try:
doc = doc or resolver.doc
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(ParseError(e, store=self.store))
if doc is None:
doc = {}
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc = self.scan(original,
resolver=self.accessor.resolver,
ctx=ctx)
self.accessor.pop_stack()
return new_doc
else:
for path, sd in self.ref_walking.iterate(doc):
try:
new_sd = self.scan(sd, resolver=resolver, ctx=ctx)
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except (KeyError, FileNotFoundError) as e:
self.errors.append(
ReferenceError(e,
store=self.store,
path=path[:],
data=sd))
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(
ParseError(e,
store=self.store,
path=path[:],
data=sd))
return doc
import pathlib
import importlib.util
from dictknife import loading
from dictknife import Accessor
from dictknife import DictWalker
spec = importlib.util.find_spec("botocore")
path = pathlib.Path(spec.origin)
if path.name == "__init__.py":
path = path.parent
d = loading.loadfile(path / ("data/sqs/2012-11-05/service-2.json"))
dst = {}
a = Accessor(make_dict=dict)
for name, sd in d["operations"].items():
path = ["operations", name]
a.assign(dst, path, sd)
ssd = a.access(d, ["shapes", sd["input"]["shape"]])
a.assign(dst, ["shapes", sd["input"]["shape"]], ssd)
if "output" in sd:
ssd = a.access(d, ["shapes", sd["output"]["shape"]])
a.assign(dst, ["shapes", sd["output"]["shape"]], ssd)
# slim-up
for path, sd in DictWalker(["documentation"]).walk(dst):
sd.pop("documentation")
loading.dumpfile(dst, format="json")
class Loader:
def __init__(self, resolver, *, store: _yaml.NodeStore):
self.resolver = resolver
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker([is_ref])
self.errors = []
self.store = store
@property
def filename(self) -> str:
return self.resolver.filename
def load(self, doc=None, resolver=None):
if not doc and doc is not None:
return doc
resolver = resolver or self.resolver
try:
doc = doc or resolver.doc
except _yaml.MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(ParseError(e, history=[resolver.filename]))
if doc is None:
doc = {}
doc, _ = self._load(doc, resolver=resolver, seen={})
return doc
def _load(self, doc, *, resolver, seen: dict):
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc, _ = self._load(original,
resolver=self.accessor.resolver,
seen=seen)
return new_doc, self.accessor.pop_stack()
else:
for path, sd in self.ref_walking.iterate(doc):
try:
uid = id(sd)
if uid in seen:
continue
seen[uid] = sd
new_sd, sresolver = self._load(sd,
resolver=resolver,
seen=seen)
if resolver.filename != sresolver.filename:
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except FileNotFoundError as e:
self.errors.append(
ResolutionError(
e,
path=path[:],
data=sd,
history=[
r.filename for r in self.accessor.stack[:-1]
],
))
except KeyError as e:
self.errors.append(
ResolutionError(
e,
path=path[:],
data=sd,
history=[r.filename for r in self.accessor.stack],
))
except _yaml.MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(
ParseError(
e,
path=path[:],
data=sd,
history=[
r.filename for r in self.accessor.stack
],
))
return doc, resolver
from dictknife import Accessor
from dictknife.pp import pp, indent
a = Accessor(OrderedDict)
d = OrderedDict()
# assign
a.assign(d, ['a', 'b', 'c'], 'v')
with indent(2, 'assign:\n'):
print(d)
pp(d)
print()
# access
with indent(2, '\naccess: ["a", "b", "c"]\n'):
print(['a', 'b', 'c'], a.access(d, ['a', 'b', 'c']))
# print(['a', 'b', 'x'], a.access(d, ['a', 'b', 'x'])) # error
# exists
with indent(2, '\nexists:\n'):
import copy # NOQA
d2 = copy.deepcopy(d)
print(['a', 'b', 'c'], a.exists(d2, ['a', 'b', 'c']))
print(['a', 'b', 'x'], a.exists(d2, ['a', 'b', 'x']))
# maybe_remove
with indent(2, '\nmaybe_remove:\n'):
import copy # NOQA
d2 = copy.deepcopy(d)
