def merge(
*,
files: list,
dst: str,
style: str, # flavor?, strategy?
strict: bool = False,
wrap: str = None,
wrap_section: str = "definitions"
):
"""merge files"""
from dictknife.langhelpers import make_dict, as_jsonpointer
from dictknife import deepmerge
if style == "ref":
dstdir = dst and os.path.dirname(dst)
r = make_dict()
seen = {}
for src in files:
d = loading.loadfile(src)
for ns, sd in d.items():
for name in sd:
if ns not in r:
r[ns] = make_dict()
seen[ns] = make_dict()
if strict and name in r[ns]:
raise RuntimeError(
"{name} is already existed, (where={where} and {where2})".format(
name=name, where=seen[ns][name], where2=src
)
)
if dst is None:
where = ""
else:
where = os.path.relpath(src, start=dstdir)
r[ns][name] = {
"$ref": "{where}#/{ns}/{name}".format(
where=where, ns=ns, name=as_jsonpointer(name)
)
}
seen[ns][name] = src
elif style == "whole":
# TODO: strict support?
data = [loading.loadfile(src) for src in files]
r = deepmerge(*data, override=True)
else:
raise RuntimeError("invalid style: {}".format(style))
if wrap is not None:
wd = make_dict()
wd["type"] = "object"
wd["properties"] = make_dict()
for name in r.get(wrap_section) or {}:
wd["properties"][name] = {
"$ref": "#/{wrap_section}/{name}".format(
wrap_section=wrap_section, name=name
)
}
r[wrap_section][wrap] = wd
loading.dumpfile(r, dst)
def test_it3(self):
from dictknife.langhelpers import make_dict
d0 = make_dict([("type", "string"), ("enum", ["C", "M", "Y", "K"])])
d1 = make_dict([("type", "string"), ("enum", ["K", "Y", "M", "C"])])
self.assertNotEqual(d0, d1)
self.assertTrue(self._callFUT(d0, d1, normalize=True))
def __init__(self, annotations):
self.doc = make_dict(definitions=make_dict())
self.definitions = self.doc["definitions"]
self.ns = NameStore()
self.cw = CommentWriter()
self.annotations = annotations or {} # Dict[string, Dict]
def _on_object(info):
r = make_dict(type="object")
r["properties"] = make_dict()
r["required"] = []
for name, child in info["children"].items():
r["properties"][name] = _on_schema(child)
if child.get("freq2") or (child["freq"] == info["freq"]):
r["required"].append(name)
if not r["required"]:
r.pop("required")
if not r["properties"]:
r.pop("properties")
return r
def run(*, src: str) -> None:
resolver = get_resolver(src)
with Migration(resolver).migrate_dryrun_and_diff() as m:
item_map = m.item_map
for k, item in list(item_map.items()):
if k == "definitions/person":
item.data["properties"] = ChainMap(make_dict(), item.data["properties"])
item.data["properties"]["value"] = {"type": "integer"}
if k == "definitions/name":
item.data = ChainMap(make_dict(), item.data)
item.resolver.assign_by_json_pointer(item.localref, item.data)
item.data["description"] = "name of something"
def run(*, src: str) -> None:
resolver = get_resolver(src)
with Migration(resolver).migrate_dryrun_and_diff() as m:
item_map = m.item_map
for k, item in list(item_map.items()):
if k == "definitions/person":
item.data["properties"] = ChainMap(make_dict(),
item.data["properties"])
item.data["properties"]["value"] = {"type": "integer"}
if k == "definitions/name":
item.data = ChainMap(make_dict(), item.data)
item.resolver.assign_by_json_pointer(item.localref, item.data)
item.data["description"] = "name of something"
def _scan(self, doc, *, resolver, seen: dict):
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc, _ = self._scan(
original, resolver=self.accessor.resolver, seen=seen
)
return new_doc, self.accessor.pop_stack()
else:
for path, sd in self.ref_walking.iterate(doc):
try:
uid = id(sd)
if uid in seen:
continue
seen[uid] = sd
new_sd, sresolver = self._scan(sd, resolver=resolver, seen=seen)
if resolver.filename != sresolver.filename:
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except (KeyError, FileNotFoundError) as e:
self.errors.append(
ReferenceError(e, store=self.store, path=path[:], data=sd)
)
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(
ParseError(e, store=self.store, path=path[:], data=sd)
)
return doc, resolver
def emit(self, resolver, doc, *, conflicted):
# side effect
d = make_dict()
for path, sd in self.ref_walking.iterate(doc):
self.replace_ref(resolver, sd)
d = deepmerge(d, doc)
for name, item in self.item_map.items():
if name == "":
continue
data = item.data
# replace: <file.yaml>#/<ref> -> #/<ref>
for path, sd in self.ref_walking.iterate(data):
if not sd["$ref"].startswith("#/"):
self.replace_ref(item.resolver, sd)
if sd["$ref"] in conflicted:
self.replace_ref(item.resolver, sd)
self.raw_accessor.assign(d, name.split("/"), data)
# adhoc paths support
will_removes = set()
paths = d.get("paths") or {}
for path, sub in list(paths.items()):
if "$ref" in sub and sub["$ref"].startswith("#/"):
related_path = tuple(sub["$ref"][2:].split("/"))
paths[path] = self.raw_accessor.access(d, related_path).copy()
will_removes.add(related_path)
for related_path in will_removes:
self.raw_accessor.maybe_remove(d, related_path)
return d
def _show_on_lifezero(d, *, path):
if hasattr(d, "keys"):
rep = make_dict()
for k, v in d.items():
if hasattr(v, "keys"):
rep[k] = f"{_describe_type(v)}@{len(v)}"
elif isinstance(v, (list, tuple)):
rep[k] = f"{_describe_type(v)}@{len(v)}"
else:
rep[k] = f"{_describe_type(v)}"
sig = json.dumps(rep, sort_keys=True, default=str)
if sig in sigmap:
return sigmap[sig]
sigmap[sig] = path_to_json_pointer(path)
return rep
elif isinstance(d, (list, tuple)):
seen = {}
for x in d:
rep = _show(x, life=0, path=path)
sig = json.dumps(rep, sort_keys=True, default=str)
if sig in seen:
continue
seen[sig] = rep
return {"$len": len(d), "$cases": list(seen.values())}
else:
return _describe_type(d)
def deepmerge(*ds, override=False, method="addtoset"):
"""deepmerge: methods in {METHODS!r}""".format(METHODS=METHODS)
if len(ds) == 0:
return make_dict()
if override:
warnings.warn(
"override option is deprecated, will be removed, near future",
category=DeprecationWarning,
)
merge = _deepmerge_replace
elif method == "addtoset":
merge = partial(_deepmerge_extend, dedup=True)
elif method == "append":
merge = partial(_deepmerge_extend, dedup=False)
elif method == "merge":
merge = _deepmerge_merge
elif method == "replace":
merge = _deepmerge_replace
else:
raise ValueError(
"unavailable method not in {METHODS!r}".format(METHODS=METHODS))
left = ds[0].__class__()
for right in ds:
if not right:
continue
left = merge(left, right)
return left
def _show(d,
*,
path,
life,
showzero_callable=showzero_callable or _show_on_lifezero):
if life == 0: # -1 is full expand
return showzero_callable(d, path=path)
if hasattr(d, "keys"):
rep = make_dict()
for k, v in d.items():
path.append(k)
if hasattr(v, "__len__") and len(v) == 0:
rep[k] = f"{_describe_type(v)}@{len(v)}"
elif hasattr(v, "keys"):
rep[k] = _show(v, life=life - 1, path=path)
elif isinstance(v, (list, tuple)):
rep[k] = _show(v, life=life - 1, path=path)
else:
rep[k] = f"{_describe_type(v)}"
path.pop()
sig = json.dumps(rep, sort_keys=True, default=str)
if sig in sigmap:
return sigmap[sig]
sigmap[sig] = path_to_json_pointer(path)
return rep
elif isinstance(d, (list, tuple)):
seen = {}
members = []
path.append("[]")
csigmap = {}
for x in d:
rep = _show(x, life=life - 1, path=path)
sig = json.dumps(rep, sort_keys=True, default=str)
if sig in seen:
members.append(csigmap[sig])
continue
rep = copy.deepcopy(rep)
for ks, sd in walker.walk(rep):
sd.pop(ks[-1])
seen[sig] = rep
csigmap[
sig] = f"{path_to_json_pointer(path)}/$cases/{len(csigmap)}"
members.append(csigmap[sig])
rep = {"$len": len(d)}
if seen:
rep["$cases"] = list(seen.values())
if members:
rep["$members"] = members
path.pop()
sig = json.dumps(rep, sort_keys=True, default=str)
if sig in sigmap:
return sigmap[sig]
sigmap[sig] = path_to_json_pointer(path)
return rep
else:
return _describe_type(d)
def make_info(self):
return {
"freq": 0,
"freq2": 0,
"type": "any",
"children": make_dict(),
"values": [],
}
def _on_array(info):
r = make_dict(type="array")
r["items"] = _on_schema(info, from_array=True)
# xxx:
if r["items"].get("type", "any") == "any":
r.pop("items")
return r
def _on_primitive(info):
r = make_dict(type=info["type"])
if "format" in info:
r["format"] = info["format"]
if info["values"]:
r["example"] = info["values"][0]
if info.get("type2") == "null":
r["nullable"] = True
return r
def make_array_schema(self, info, parent):
self.cw.write(info["name"] + "[]", info, parent=parent)
d = make_dict(type="array")
d["items"] = self.make_schema(info, parent=info, fromarray=True)
schema_name = self.resolve_name(info, suffix="[]")
self.definitions[schema_name] = d
if info["ref"] + "[]" in self.annotations:
d.update(self.annotations[info["ref"] + "[]"])
d.pop("name", None)
return {"$ref": "#/definitions/{name}".format(name=schema_name)}
def __init__(self,
resolver,
*,
make_dict=make_dict,
dump_options=None,
transform=None):
self.resolver = resolver
self.item_map = make_dict()
self.make_dict = make_dict
self.dump_options = dump_options or {}
self._transform = transform
def transform(doc):
heavy_defs = ["definitions", "schemas", "responses", "parameters", "paths"]
r = make_dict()
for k, v in doc.items():
if k in heavy_defs:
continue
r[k] = v
for k in heavy_defs:
if k in doc:
r[k] = doc[k]
return r
def transform(doc):
heavy_defs = ["definitions", "schemas", "responses", "parameters", "paths"]
r = make_dict()
for k, v in doc.items():
if k in heavy_defs:
continue
r[k] = v
for k in heavy_defs:
if k in doc:
r[k] = doc[k]
return r
def make_primitive_schema(self, info):
d = make_dict(type=info["type"])
if "format" in info:
d["format"] = info["format"]
if info["values"]:
d["example"] = info["values"][0]
if info.get("type2") == "null":
d["x-nullable"] = True
if info["ref"] in self.annotations:
d.update(self.annotations[info["ref"]])
d.pop("name", None)
return d
def run(*, src: str) -> None:
resolver = get_resolver(src)
accessor = CachedItemAccessor(resolver)
item_map = make_dict()
scanner = Scanner(accessor, item_map, strict=True)
scanner.scan(resolver.doc)
resolvers = set()
for k, item in list(item_map.items()):
if k == "definitions/person":
item.data["properties"] = ChainMap(make_dict(), item.data["properties"])
item.data["properties"]["value"] = {"type": "integer"}
if k == "definitions/name":
item.data = ChainMap(make_dict(), item.data)
item.resolver.assign_by_json_pointer(item.localref, item.data)
item.data["description"] = "name of something"
resolvers.add(item.resolver)
for r in resolvers:
print(os.path.relpath(r.name, start=os.getcwd()))
for line in diff(before_data(r.doc), after_data(r.doc)):
print(line)
def transform(doc, *, sort_keys=False):
"""reorder"""
heavy_defs = ["definitions", "schemas", "responses", "parameters", "paths"]
r = make_dict()
for k, v in doc.items():
if k in heavy_defs:
continue
r[k] = v
for k in heavy_defs:
if k in doc:
r[k] = doc[k]
if sort_keys:
r = str_dict(r) # side effect
return r
def run(*, src: str) -> None:
resolver = get_resolver(src)
accessor = CachedItemAccessor(resolver)
item_map = make_dict()
scanner = Scanner(accessor, item_map, strict=True)
scanner.scan(resolver.doc)
resolvers = set()
for k, item in list(item_map.items()):
if k == "definitions/person":
item.data["properties"] = ChainMap(make_dict(),
item.data["properties"])
item.data["properties"]["value"] = {"type": "integer"}
if k == "definitions/name":
item.data = ChainMap(make_dict(), item.data)
item.resolver.assign_by_json_pointer(item.localref, item.data)
item.data["description"] = "name of something"
resolvers.add(item.resolver)
for r in resolvers:
print(os.path.relpath(r.name, start=os.getcwd()))
for line in diff(before_data(r.doc), after_data(r.doc)):
print(line)
def make_object_schema(self, info, parent, fromarray=False):
if not fromarray:
self.cw.write(info["name"], info, parent=parent)
d = make_dict(type="object")
d["properties"] = make_dict()
props = d["properties"]
for name, value in info["children"].items():
props[name] = self.make_schema(value, parent=info)
required = [
name for name, f in info["children"].items()
if (f.get("freq2") or f["freq"]) == info["freq"]
]
if required:
d["required"] = required
if info.get("type2") == "null":
d["x-nullable"] = True
schema_name = self.resolve_name(info, fromarray=fromarray)
self.definitions[schema_name] = d
if info["ref"] in self.annotations:
d.update(self.annotations[info["ref"]])
d.pop("name", None)
return {"$ref": "#/definitions/{name}".format(name=schema_name)}
def shape(
*,
files,
input_format,
output_format,
squash,
skiplist,
separator,
with_type,
with_example,
full
):
"""shape"""
from dictknife import shape
dataset = []
for f in files:
with _open(f) as rf:
d = loading.load(rf, format=input_format)
if squash:
dataset.extend(d)
else:
dataset.append(d)
rows = shape(dataset, squash=True, skiplist=skiplist, separator=separator)
r = []
for row in rows:
d = make_dict()
d["path"] = row.path
if with_type:
typenames = [t.__name__ for t in row.type]
d["type"] = typenames[0] if len(typenames) == 1 else typenames
if with_example:
if full:
d["example"] = row.example
elif not any(t in (list, dict) for t in row.type):
d["example"] = row.example
elif output_format in ("csv", "tsv"):
d["example"] = "" # xxx
r.append(d)
if output_format is None:
for d in r:
print(*d.values())
else:
loading.dumpfile(r, format=output_format)
def cat(*,
files,
dst,
format,
input_format,
output_format,
sort_keys,
encoding=None,
errors=None,
size=None,
slurp=False,
extra=None,
merge_method="addtoset"):
from dictknife import deepmerge
input_format = input_format or format
d = make_dict()
with contextlib.ExitStack() as s:
for f in files:
logger.debug("merge: %s", f)
opener = loading.get_opener(filename=f,
format=input_format,
default=_open)
rf = s.enter_context(opener(f, encoding=encoding, errors=errors))
if slurp:
sd = (loading.loads(line, format=input_format) for line in rf)
else:
sd = loading.load(rf, format=input_format, errors=errors)
if size is not None:
sd = itertools.islice(sd, size)
if hasattr(sd, "keys"):
d = deepmerge(d, sd, method=merge_method)
elif len(files) == 1:
d = sd
else:
if not isinstance(d, (list, tuple)):
d = [d] if d else []
d = deepmerge(d, sd, method=merge_method)
loading.dumpfile(d,
dst,
format=output_format or format,
sort_keys=sort_keys,
extra=extra)
def emit_environ(structure, make_dict, parse):
if hasattr(structure, "keys"):
d = make_dict()
for k, v in structure.items():
name, fn = parse(v)
emitted = emit_environ(name, make_dict=make_dict, parse=parse)
if emitted is None:
continue
if fn is not None:
emitted = fn(emitted)
d[k] = emitted
return d
elif isinstance(structure, (list, tuple)):
return [
emit_environ(x, make_dict=make_dict, parse=parse)
for x in structure
]
else:
return os.environ.get(structure)
def load(fp,
*,
loader=None,
errors=None,
make_dict=make_dict,
null_value="null",
**kwargs):
keys = None
while keys is None:
line = next(fp)
if "|" in line:
keys = [tok.strip() for tok in line.strip("|\n").split("|")]
maybe_nums = None
while maybe_nums is None:
line = next(fp)
if "|" in line:
maybe_nums = [
tok.rstrip().endswith(":") and not tok.lstrip().startswith(":")
for tok in line.strip("|\n").split("|")
]
for line in fp:
if "|" not in line:
continue
row = make_dict()
for name, maybe_num, tok in zip(keys, maybe_nums,
line.strip("|\n").split("|")):
val = tok.strip()
if not val:
continue
elif val == null_value:
row[name] = None
elif maybe_num:
try:
if "." in val:
row[name] = float(val)
else:
row[name] = int(val)
except ValueError:
row[name] = val
else:
row[name] = val
yield row
def expand(self, doc=None, resolver=None, ctx=None):
doc = doc or self.resolver.doc
resolver = resolver or self.resolver
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc = self.expand(original,
resolver=self.accessor.resolver,
ctx=ctx)
self.accessor.pop_stack()
return new_doc
else:
for path, sd in self.ref_walking.iterate(doc):
new_sd = self.expand(sd, resolver=resolver, ctx=ctx)
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
return doc
def __next__(self):
if self.line_num == 0:
# Used only for its side effect.
self.fieldnames
row = next(self.reader)
self.line_num = self.reader.line_num
# unlike the basic reader, we prefer not to return blanks,
# because we will typically wind up with a dict full of None
# values
while row == []:
row = next(self.reader)
d = make_dict(zip(self.fieldnames, row))
lf = len(self.fieldnames)
lr = len(row)
if lf < lr:
d[self.restkey] = row[lf:]
elif lf > lr:
for key in self.fieldnames[lr:]:
d[key] = self.restval
return d
def scan(self, doc=None, resolver=None, ctx=None):
resolver = resolver or self.resolver
try:
doc = doc or resolver.doc
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(ParseError(e, store=self.store))
if doc is None:
doc = {}
if "$ref" in doc:
original = self.accessor.access(doc["$ref"])
new_doc = self.scan(original,
resolver=self.accessor.resolver,
ctx=ctx)
self.accessor.pop_stack()
return new_doc
else:
for path, sd in self.ref_walking.iterate(doc):
try:
new_sd = self.scan(sd, resolver=resolver, ctx=ctx)
container = self.accessing.access(doc, path[:-1])
if not hasattr(container, "parents"):
container = ChainMap(make_dict(), container)
container.update(new_sd)
self.accessing.assign(doc, path[:-1], container)
except (KeyError, FileNotFoundError) as e:
self.errors.append(
ReferenceError(e,
store=self.store,
path=path[:],
data=sd))
except MarkedYAMLError as e:
if e.problem_mark is not None:
self.errors.append(
ParseError(e,
store=self.store,
path=path[:],
data=sd))
return doc
def select(self, ref, *, doc=None, a=Accessor(), r=None):
ref_wrap = ref if self.wrap else None
doc = doc or self.resolver.doc
r = r or self.r
if r is None:
r = make_dict() # too many?
if "@" in ref:
ref, ref_wrap = ref.split("@", 1)
if ref:
accessed = access_by_json_pointer(doc, ref, guess=True)
else:
accessed = doc
if ref_wrap is None:
return accessed
wrap_path = json_pointer_to_path(ref_wrap)
a.assign(r, wrap_path, accessed)
return r
def select(
*,
src: str,
dst: str,
refs,
unwrap,
wrap,
input_format: str,
output_format: str,
format: str,
):
from dictknife.jsonknife import Expander
from dictknife.jsonknife.accessor import assign_by_json_pointer
from dictknife.jsonknife import get_resolver
input_format = input_format or format
resolver = get_resolver(src)
expander = Expander(resolver)
if unwrap and not refs:
refs = []
refs.append(unwrap)
if not refs:
d = expander.expand()
else:
d = make_dict()
for ref in refs:
ref_wrap = wrap
if "@" in ref:
ref, ref_wrap = ref.split("@", 1)
extracted = expander.expand_subpart(expander.access(ref))
if ref_wrap:
assign_by_json_pointer(d, ref_wrap, extracted)
else:
d = deepmerge(d, extracted)
loading.dumpfile(d, dst, format=output_format or format)
def _extract(state, d, *, path=None):
path = path or []
typ = d.get("type")
if "properties" in d or typ == "object":
return _extract_object(state,
d.get("properties") or {},
r=make_dict(),
path=path)
elif typ == "array":
return _extract_array(state, d["items"], r=[], path=path)
elif "example" in d:
if isinstance(d["example"], (list, tuple)) and d["type"] != "array":
size = min(state.limit, len(d["example"]))
state.max_examples = max(state.max_examples, size)
return d["example"][min(state.i, len(d["example"]) - 1)]
return d["example"]
elif "enum" in d:
size = min(state.limit, len(d["enum"]))
state.max_examples = max(state.max_examples, size)
return d["enum"][min(state.i, len(d["enum"]) - 1)]
elif "default" in d:
return d["default"]
else:
return "<>"
def __init__(self, resolver):
self.resolver = resolver
self.accessor = CachedItemAccessor(resolver)
self.item_map = make_dict()
def transform(
d,
*,
default_content_type="application/json",
is_specific_header=is_specific_header,
get_value=get_value,
with_response_type=True,
with_request_type=True,
with_cookies=True,
include_all=False,
):
r = make_dict()
a = Accessor()
for path, methods in d.items():
for method, entries in methods.items():
d = {"description": ""}
seen_parameters = defaultdict(set)
request_bodies: t.List[dict] = []
response_bodies_dict: t.Dict[t.Tuple[int, str], dict] = defaultdict(list)
for e in entries:
# request
# params :: path,query,header,cookie
parameters = []
for param_type, k, enabled in [
("query", "queryString", True),
("header", "headers", True),
("cookie", "cookies", with_cookies),
]:
if not enabled:
continue
seen = seen_parameters[k]
for h in e["request"][k]:
if h["name"] in seen:
continue
seen.add(h["name"])
if include_all or is_specific_header(h["name"], h["value"]):
parameters.append(
{
"name": h["name"],
"in": param_type,
"example": get_value(
h["name"], h["value"]
), # masking?
}
)
if parameters:
d["parameters"] = parameters
if e["request"].get("postData"):
post_data = e["request"]["postData"]
content_type = post_data["mimeType"].split(";", 1)[0]
if content_type.endswith("/json") and with_request_type:
request_bodies.append(
loading.loads(post_data["text"], format="json")
)
# response
status = e["response"]["status"]
if status == 304:
status = 200 # not modified -> ok
content_type = e["response"]["content"].get("mimeType")
if content_type is None:
for h in e["response"]["headers"]:
if h["name"].lower() == "content-type":
content_type = h["value"]
break
else:
content_type = default_content_type
# "application/json; charset=utf-8" -> "application/json"
content_type = content_type.split(";", 1)[0]
schema = {}
if content_type.startswith("text/"):
a.assign(schema, ["type"], "string")
elif content_type.endswith("/json") and with_response_type:
response_bodies_dict[(status, content_type)].append(
loading.loads(e["response"]["content"]["text"], format="json")
)
a.assign(
d,
["responses", status],
{
"description": e["response"]["statusText"],
"content": {content_type: {"schema": schema}},
},
)
if request_bodies:
detector = schemalib.Detector()
info = None
for body in request_bodies:
info = detector.detect(body, name="")
a.assign(d, ["requestBody"], schemalib.makeschema_from_info(info))
if response_bodies_dict:
for (status, content_type), bodies in response_bodies_dict.items():
detector = schemalib.Detector()
info = None
for body in bodies:
info = detector.detect(body, name="")
a.assign(
d,
["responses", status, "content", content_type, "schema"],
schemalib.makeschema_from_info(info),
)
a.assign(r, ["paths", path, method.lower()], d)
return r
def __init__(self, resolver, *, make_dict=make_dict):
self.resolver = resolver
self.item_map = make_dict()
self.make_dict = make_dict
