def run(*, src: str, schema: str) -> None:
with open(schema) as rf:
schema = yaml.load(rf)
with open(src) as rf:
loader = Loader(rf)
try:
assert loader.check_data()
data = loader.get_data()
finally:
loader.dispose()
jsonschema.Draft4Validator.check_schema(schema)
validator = jsonschema.Draft4Validator(schema)
for err in validator.iter_errors(data):
print("E", err)
a = Accessor()
path = list(err.path)
ob = a.access(data, path[:-1])
ev = mem[id(ob)]
for kev, vev in ev.value:
if kev.value == path[-1]:
print("----------------------------------------")
print(str(vev.start_mark).lstrip())
lineno = vev.start_mark.line + 1
with open(src) as rf:
for i, line in enumerate(rf, 1):
if lineno == i:
print(f" {i:02d}: -> {line}", end="")
else:
print(f" {i:02d}: {line}", end="")
break
def __init__(self, resolver, *, store: _yaml.NodeStore):
self.resolver = resolver
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker([is_ref])
self.errors = []
self.store = store
class Expander:
def __init__(self, resolver):
self.resolver = resolver
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker(["$ref"])
self.errors = []
def expand(self, doc=None, resolver=None, ctx=None):
doc = doc or self.resolver.doc
resolver = resolver or self.resolver
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc = self.expand(original,
resolver=self.accessor.resolver,
ctx=ctx)
self.accessor.pop_stack()
return new_doc
else:
for path, sd in self.ref_walking.iterate(doc):
try:
new_sd = self.expand(sd, resolver=resolver, ctx=ctx)
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except Exception as e:
self.errors.append(ReferenceError(e, path=path[:],
data=sd))
return doc
def __init__(self, resolver, *, store: Store):
self.resolver = resolver
self.store = store
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker(["$ref"])
self.errors = []
class Scaner:
def __init__(self, resolver, *, store: Store):
self.resolver = resolver
self.store = store
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker([is_ref])
self.errors = []
def scan(self, doc=None, resolver=None):
if not doc and doc is not None:
return doc
resolver = resolver or self.resolver
try:
doc = doc or resolver.doc
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(ParseError(e, store=self.store))
if doc is None:
doc = {}
doc, _ = self._scan(doc, resolver=resolver, seen={})
return doc
def _scan(self, doc, *, resolver, seen: dict):
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc, _ = self._scan(
original, resolver=self.accessor.resolver, seen=seen
)
return new_doc, self.accessor.pop_stack()
else:
for path, sd in self.ref_walking.iterate(doc):
try:
uid = id(sd)
if uid in seen:
continue
seen[uid] = sd
new_sd, sresolver = self._scan(sd, resolver=resolver, seen=seen)
if resolver.filename != sresolver.filename:
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except (KeyError, FileNotFoundError) as e:
self.errors.append(
ReferenceError(e, store=self.store, path=path[:], data=sd)
)
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(
ParseError(e, store=self.store, path=path[:], data=sd)
)
return doc, resolver
class Emitter:
def __init__(self, accessor, item_map):
self.raw_accessor = Accessor()
self.accessor = accessor
self.item_map = item_map
@reify
def ref_walking(self):
return DictWalker([is_ref])
def get_item_by_globalref(self, globalref):
return self.accessor.cache[globalref]
def get_item_by_localref(self, localref):
return self.item_map[localref]
def emit(self, resolver, doc, *, conflicted):
# side effect
d = make_dict()
for path, sd in self.ref_walking.iterate(doc):
self.replace_ref(resolver, sd)
d = deepmerge(d, doc)
for name, item in self.item_map.items():
if name == "":
continue
data = item.data
# replace: <file.yaml>#/<ref> -> #/<ref>
for path, sd in self.ref_walking.iterate(data):
if not sd["$ref"].startswith("#/"):
self.replace_ref(item.resolver, sd)
if sd["$ref"] in conflicted:
self.replace_ref(item.resolver, sd)
self.raw_accessor.assign(d, name.split("/"), data)
# adhoc paths support
will_removes = set()
paths = d.get("paths") or {}
for path, sub in list(paths.items()):
if "$ref" in sub and sub["$ref"].startswith("#/"):
related_path = tuple(sub["$ref"][2:].split("/"))
paths[path] = self.raw_accessor.access(d, related_path).copy()
will_removes.add(related_path)
for related_path in will_removes:
self.raw_accessor.maybe_remove(d, related_path)
return d
def replace_ref(self, resolver, sd):
filename, _, pointer = resolver.resolve_pathset(sd["$ref"])
related = self.get_item_by_globalref((filename, pointer))
new_ref = "#/{}".format(related.localref)
if sd["$ref"] != new_ref:
logger.debug(
"fix ref: %r -> %r (where=%r)", sd["$ref"], new_ref, resolver.filename
)
sd["$ref"] = new_ref
class Scaner:
def __init__(self, resolver, *, store: Store):
self.resolver = resolver
self.store = store
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker(["$ref"])
self.errors = []
def scan(self, doc=None, resolver=None, ctx=None):
resolver = resolver or self.resolver
try:
doc = doc or resolver.doc
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(ParseError(e, store=self.store))
if doc is None:
doc = {}
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc = self.scan(original,
resolver=self.accessor.resolver,
ctx=ctx)
self.accessor.pop_stack()
return new_doc
else:
for path, sd in self.ref_walking.iterate(doc):
try:
new_sd = self.scan(sd, resolver=resolver, ctx=ctx)
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except (KeyError, FileNotFoundError) as e:
self.errors.append(
ReferenceError(e,
store=self.store,
path=path[:],
data=sd))
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(
ParseError(e,
store=self.store,
path=path[:],
data=sd))
return doc
def assign_by_json_pointer(doc, query, v, *, accessor=Accessor(), guess=False):
if query == "":
return doc
try:
path = json_pointer_to_path(query)
return accessor.assign(doc, path, v)
except KeyError:
if guess:
new_path = []
has_integer = False
for p in path:
if p.isdigit():
new_path.append(int(p))
has_integer = True
else:
new_path.append(p)
if has_integer:
logger.debug("jsonpointer: %r is notfound. including integer?",
query)
try:
return accessor.assign(doc, new_path, v)
except KeyError:
pass
raise KeyError(query)
def __init__(self,
resolver,
*,
accessor=Accessor(),
wrap_exception=wrap_exception):
self.stack = [resolver]
self.accessor = accessor
self.wrap_exception = wrap_exception
def main(*, src: str) -> None:
precompile_ref_walker = DictWalker(["$precompile-ref"])
accessor = Accessor()
def onload(d, subresolver):
for path, sd in precompile_ref_walker.walk(d):
sdoc, query = subresolver.resolve(sd.pop("$precompile-ref"))
sresolved = access_by_json_pointer(sdoc.doc, query)
accessor.assign(d, path[:-1], sresolved)
resolver = get_resolver_from_filename(src, onload=onload)
d = Bundler(resolver).bundle()
loading.dumpfile(d)
def run(pkgname: str) -> None:
p = subprocess.run(["pyinspect", "list", pkgname],
text=True,
stdout=subprocess.PIPE)
a = Accessor()
r = a.make_dict()
for module_name in p.stdout.split():
try:
m = importlib.import_module(module_name)
except Exception as e:
print(f"\x1b[32m!! {e!r} \x1b[0m", file=sys.stderr)
for name, val in m.__dict__.items():
if inspect.isclass(val):
a.assign(r, [module_name, name], inspect.getdoc(val))
elif inspect.isfunction(val):
a.assign(r, [module_name, name], inspect.getdoc(val))
loading.dumpfile(r)
def get(name, context, default=strict_default, accessor=Accessor()):
try:
if "#/" not in name:
return getattr(context.scope, name)
else:
subname, path = name.split("#/", 2)
root = getattr(context.scope, subname)
return accessor.access(root, path.split("/"))
except AttributeError as e:
if default is strict_default:
vs = sorted(k for k in get_locals(context.scope).keys()
if not k.startswith("__"))
msg = "{e}\n accessible variables: {vs}".format(e=e, vs=vs)
raise EvalVariableError(msg)
return default
except KeyError as e:
if default is strict_default:
raise EvalVariableError("KeyError: {}".format(name))
return default
def select(self, ref, *, doc=None, a=Accessor(), r=None):
ref_wrap = ref if self.wrap else None
doc = doc or self.resolver.doc
r = r or self.r
if r is None:
r = make_dict() # too many?
if "@" in ref:
ref, ref_wrap = ref.split("@", 1)
if ref:
accessed = access_by_json_pointer(doc, ref, guess=True)
else:
accessed = doc
if ref_wrap is None:
return accessed
wrap_path = json_pointer_to_path(ref_wrap)
a.assign(r, wrap_path, accessed)
return r
def __init__(self, accessor, item_map):
self.raw_accessor = Accessor()
self.accessor = accessor
self.item_map = item_map
class Resolver(object):
def __init__(self):
self.route_name_store = RouteNameStore()
self.accessor = Accessor()
def resolve_view_path(self, rootdata, d):
return d["operationId"]
def resolve_module_name(self, view_path):
return view_path.rsplit(".", 1)[0]
def resolve_route_name(self, module_name, pattern, route_name=None):
if route_name is not None:
return route_name
k = (module_name, pattern)
self.route_name_store[k] = module_name
return self.route_name_store[k]
def resolve_ref(self, fulldata, d):
# todo: support quoted "/"
if "$ref" not in d:
return d
path = d["$ref"][len("#/"):].split("/")
name = path[-1]
parent = self.accessor.maybe_access_container(fulldata, path)
if parent is None:
sys.stderr.write("\t{!r} is not found\n".format(d["$ref"]))
return d
return self.resolve_ref(fulldata, parent[name])
rename_dict = {
"path": "request.matchdict:",
"query": "request.GET:",
"formData": "request.POST:",
"body": "request.json_body:"
}
def view_docstring(self, fulldata, d, rename_dict=rename_dict):
if "parameters" not in d:
return None
parameters = defaultdict(list)
for p in d["parameters"]:
p = self.resolve_ref(fulldata, p)
parameters[p["in"]].append(p)
# "query", "header", "path", "formData" or "body"
m = Module()
body = parameters.pop("body", None)
for name, vs in parameters.items():
pyramid_name = rename_dict.get(name) or name
m.stmt(pyramid_name)
m.stmt("")
with m.scope():
for v in vs:
name = v.get("name", "")
description = v.get("description", "-")
if description:
description = " {}".format(description)
extra = v.copy()
extra.pop("in", None)
extra.pop("name", None)
extra.pop("description", None)
if extra:
extra_string = " `{}`".format(json.dumps(extra))
else:
extra_string = ""
m.stmt("* {name!r}{description}{extra}".format(
name=name, description=description,
extra=extra_string))
if body:
m.stmt("")
m.stmt(rename_dict["body"])
m.stmt("\n```")
with m.scope():
schema = self.resolve_ref(fulldata, body[0]["schema"])
json_str = json.dumps(schema, ensure_ascii=False, indent=2)
for line in json_str.split("\n"):
m.stmt(line)
m.stmt("```")
return str(m)
import pathlib
import importlib.util
from dictknife import loading
from dictknife import Accessor
from dictknife import DictWalker
spec = importlib.util.find_spec("botocore")
path = pathlib.Path(spec.origin)
if path.name == "__init__.py":
path = path.parent
d = loading.loadfile(path / ("data/sqs/2012-11-05/service-2.json"))
dst = {}
a = Accessor(make_dict=dict)
for name, sd in d["operations"].items():
path = ["operations", name]
a.assign(dst, path, sd)
ssd = a.access(d, ["shapes", sd["input"]["shape"]])
a.assign(dst, ["shapes", sd["input"]["shape"]], ssd)
if "output" in sd:
ssd = a.access(d, ["shapes", sd["output"]["shape"]])
a.assign(dst, ["shapes", sd["output"]["shape"]], ssd)
# slim-up
for path, sd in DictWalker(["documentation"]).walk(dst):
sd.pop("documentation")
loading.dumpfile(dst, format="json")
def _callFUT(self, d, k, v):
from dictknife import Accessor
a = Accessor(make_dict=dict)
return a.assign(d, k, v)
def _callFUT(self, d, path):
from dictknife import Accessor
a = Accessor(make_dict=dict)
return a.maybe_access(d, path)
def __init__(self):
self.route_name_store = RouteNameStore()
self.accessor = Accessor()
def accessor(self):
return Accessor()
def assign(self, path, value, *, doc=None, a=Accessor()):
if doc is None:
doc = self.doc
return a.assign(doc, path, value)
def maybe_access(self, path, *, doc=None, a=Accessor()):
if doc is None:
doc = self.doc
return a.maybe_access(doc, path)
def mget(d, path, *, a=Accessor()):
return a.maybe_access(d, json_pointer_to_path(path))
def transform(
d,
*,
default_content_type="application/json",
is_specific_header=is_specific_header,
get_value=get_value,
with_response_type=True,
with_request_type=True,
with_cookies=True,
include_all=False,
):
r = make_dict()
a = Accessor()
for path, methods in d.items():
for method, entries in methods.items():
d = {"description": ""}
seen_parameters = defaultdict(set)
request_bodies: t.List[dict] = []
response_bodies_dict: t.Dict[t.Tuple[int, str],
dict] = defaultdict(list)
for e in entries:
# request
# params :: path,query,header,cookie
parameters = []
for param_type, k, enabled in [
("query", "queryString", True),
("header", "headers", True),
("cookie", "cookies", with_cookies),
]:
if not enabled:
continue
seen = seen_parameters[k]
for h in e["request"][k]:
if h["name"] in seen:
continue
seen.add(h["name"])
if include_all or is_specific_header(
h["name"], h["value"]):
parameters.append({
"name":
h["name"],
"in":
param_type,
"example":
get_value(h["name"], h["value"]), # masking?
})
if parameters:
d["parameters"] = parameters
if e["request"].get("postData"):
post_data = e["request"]["postData"]
content_type = post_data["mimeType"].split(";", 1)[0]
if content_type.endswith("/json") and with_request_type:
request_bodies.append(
loading.loads(post_data["text"], format="json"))
# response
status = e["response"]["status"]
if status == 304:
status = 200 # not modified -> ok
content_type = e["response"]["content"].get("mimeType")
if content_type is None:
for h in e["response"]["headers"]:
if h["name"].lower() == "content-type":
content_type = h["value"]
break
else:
content_type = default_content_type
# "application/json; charset=utf-8" -> "application/json"
content_type = content_type.split(";", 1)[0]
schema = {}
if content_type.startswith("text/"):
a.assign(schema, ["type"], "string")
elif content_type.endswith("/json") and with_response_type:
response_bodies_dict[(status, content_type)].append(
loading.loads(e["response"]["content"]["text"],
format="json"))
a.assign(
d,
["responses", status],
{
"description": e["response"]["statusText"],
"content": {
content_type: {
"schema": schema
}
},
},
)
if request_bodies:
detector = schemalib.Detector()
info = None
for body in request_bodies:
info = detector.detect(body, name="")
a.assign(d, ["requestBody"],
schemalib.makeschema_from_info(info))
if response_bodies_dict:
for (status,
content_type), bodies in response_bodies_dict.items():
detector = schemalib.Detector()
info = None
for body in bodies:
info = detector.detect(body, name="")
a.assign(
d,
[
"responses", status, "content", content_type,
"schema"
],
schemalib.makeschema_from_info(info),
)
a.assign(r, ["paths", path, method.lower()], d)
return r
def __init__(self, resolver):
self.resolver = resolver
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker(["$ref"])
from collections import OrderedDict
from dictknife import Accessor
from dictknife.pp import pp, indent
a = Accessor(OrderedDict)
d = OrderedDict()
# assign
a.assign(d, ['a', 'b', 'c'], 'v')
with indent(2, 'assign:\n'):
print(d)
pp(d)
print()
# access
with indent(2, '\naccess: ["a", "b", "c"]\n'):
print(['a', 'b', 'c'], a.access(d, ['a', 'b', 'c']))
# print(['a', 'b', 'x'], a.access(d, ['a', 'b', 'x'])) # error
# exists
with indent(2, '\nexists:\n'):
import copy # NOQA
d2 = copy.deepcopy(d)
print(['a', 'b', 'c'], a.exists(d2, ['a', 'b', 'c']))
print(['a', 'b', 'x'], a.exists(d2, ['a', 'b', 'x']))
# maybe_remove
with indent(2, '\nmaybe_remove:\n'):
import copy # NOQA
d2 = copy.deepcopy(d)
class Loader:
def __init__(self, resolver, *, store: _yaml.NodeStore):
self.resolver = resolver
self.accessor = StackedAccessor(resolver)
self.accessing = Accessor()
self.ref_walking = DictWalker([is_ref])
self.errors = []
self.store = store
@property
def filename(self) -> str:
return self.resolver.filename
def load(self, doc=None, resolver=None):
if not doc and doc is not None:
return doc
resolver = resolver or self.resolver
try:
doc = doc or resolver.doc
except _yaml.MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(ParseError(e, history=[resolver.filename]))
if doc is None:
doc = {}
doc, _ = self._load(doc, resolver=resolver, seen={})
return doc
def _load(self, doc, *, resolver, seen: dict):
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc, _ = self._load(original,
resolver=self.accessor.resolver,
seen=seen)
return new_doc, self.accessor.pop_stack()
else:
for path, sd in self.ref_walking.iterate(doc):
try:
uid = id(sd)
if uid in seen:
continue
seen[uid] = sd
new_sd, sresolver = self._load(sd,
resolver=resolver,
seen=seen)
if resolver.filename != sresolver.filename:
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except FileNotFoundError as e:
self.errors.append(
ResolutionError(
e,
path=path[:],
data=sd,
history=[
r.filename for r in self.accessor.stack[:-1]
],
))
except KeyError as e:
self.errors.append(
ResolutionError(
e,
path=path[:],
data=sd,
history=[r.filename for r in self.accessor.stack],
))
except _yaml.MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(
ParseError(
e,
path=path[:],
data=sd,
history=[
r.filename for r in self.accessor.stack
],
))
return doc, resolver
def maybe_remove_by_json_pointer(doc, query, *, accessor=Accessor()):
if query == "":
return
path = json_pointer_to_path(query)
return accessor.maybe_remove(doc, path)
def __init__(self, resolver, accessor=Accessor()):
self.stack = [resolver]
self.accessor = accessor
"version": "3.8",
"downloads": 500
},
]
z_packages = [
{
"version": "3.7",
"downloads": 20000
},
{
"version": "3.8",
"downloads": 500000
},
]
mget = Accessor().maybe_access
rows = list(join(x_packages, y_packages, on="version",
how=how_full_outer_join))
rows2 = list(
join(
rows,
z_packages,
right_on="version",
left_on=lambda p: mget(p, [0, "version"]) or mget(p, [1, "version"]),
how=how_full_outer_join,
))
results = [{
"version":
mget(x, ["version"]) or mget(y, ["version"]) or mget(z, ["version"]),
"x_downloads":
mget(x, ["downloads"]) or "",
def transform(
d,
*,
default_content_type="application/json",
is_specific_header=is_specific_header,
get_value=get_value,
with_response_type=True,
with_request_type=True,
with_cookies=True,
include_all=False,
):
r = make_dict()
a = Accessor()
for path, methods in d.items():
for method, entries in methods.items():
d = {"description": ""}
seen_parameters = defaultdict(set)
request_bodies: t.List[dict] = []
response_bodies_dict: t.Dict[t.Tuple[int, str], dict] = defaultdict(list)
for e in entries:
# request
# params :: path,query,header,cookie
parameters = []
for param_type, k, enabled in [
("query", "queryString", True),
("header", "headers", True),
("cookie", "cookies", with_cookies),
]:
if not enabled:
continue
seen = seen_parameters[k]
for h in e["request"][k]:
if h["name"] in seen:
continue
seen.add(h["name"])
if include_all or is_specific_header(h["name"], h["value"]):
parameters.append(
{
"name": h["name"],
"in": param_type,
"example": get_value(
h["name"], h["value"]
), # masking?
}
)
if parameters:
d["parameters"] = parameters
if e["request"].get("postData"):
post_data = e["request"]["postData"]
content_type = post_data["mimeType"].split(";", 1)[0]
if content_type.endswith("/json") and with_request_type:
request_bodies.append(
loading.loads(post_data["text"], format="json")
)
# response
status = e["response"]["status"]
if status == 304:
status = 200 # not modified -> ok
content_type = e["response"]["content"].get("mimeType")
if content_type is None:
for h in e["response"]["headers"]:
if h["name"].lower() == "content-type":
content_type = h["value"]
break
else:
content_type = default_content_type
# "application/json; charset=utf-8" -> "application/json"
content_type = content_type.split(";", 1)[0]
schema = {}
if content_type.startswith("text/"):
a.assign(schema, ["type"], "string")
elif content_type.endswith("/json") and with_response_type:
response_bodies_dict[(status, content_type)].append(
loading.loads(e["response"]["content"]["text"], format="json")
)
a.assign(
d,
["responses", status],
{
"description": e["response"]["statusText"],
"content": {content_type: {"schema": schema}},
},
)
if request_bodies:
detector = schemalib.Detector()
info = None
for body in request_bodies:
info = detector.detect(body, name="")
a.assign(d, ["requestBody"], schemalib.makeschema_from_info(info))
if response_bodies_dict:
for (status, content_type), bodies in response_bodies_dict.items():
detector = schemalib.Detector()
info = None
for body in bodies:
info = detector.detect(body, name="")
a.assign(
d,
["responses", status, "content", content_type, "schema"],
schemalib.makeschema_from_info(info),
)
a.assign(r, ["paths", path, method.lower()], d)
return r
