def merge( *, files: list, dst: str, style: str, # flavor?, strategy? strict: bool = False, wrap: str = None, wrap_section: str = "definitions" ): """merge files""" from dictknife.langhelpers import make_dict, as_jsonpointer from dictknife import deepmerge if style == "ref": dstdir = dst and os.path.dirname(dst) r = make_dict() seen = {} for src in files: d = loading.loadfile(src) for ns, sd in d.items(): for name in sd: if ns not in r: r[ns] = make_dict() seen[ns] = make_dict() if strict and name in r[ns]: raise RuntimeError( "{name} is already existed, (where={where} and {where2})".format( name=name, where=seen[ns][name], where2=src ) ) if dst is None: where = "" else: where = os.path.relpath(src, start=dstdir) r[ns][name] = { "$ref": "{where}#/{ns}/{name}".format( where=where, ns=ns, name=as_jsonpointer(name) ) } seen[ns][name] = src elif style == "whole": # TODO: strict support? data = [loading.loadfile(src) for src in files] r = deepmerge(*data, override=True) else: raise RuntimeError("invalid style: {}".format(style)) if wrap is not None: wd = make_dict() wd["type"] = "object" wd["properties"] = make_dict() for name in r.get(wrap_section) or {}: wd["properties"][name] = { "$ref": "#/{wrap_section}/{name}".format( wrap_section=wrap_section, name=name ) } r[wrap_section][wrap] = wd loading.dumpfile(r, dst)
def test_it3(self): from dictknife.langhelpers import make_dict d0 = make_dict([("type", "string"), ("enum", ["C", "M", "Y", "K"])]) d1 = make_dict([("type", "string"), ("enum", ["K", "Y", "M", "C"])]) self.assertNotEqual(d0, d1) self.assertTrue(self._callFUT(d0, d1, normalize=True))
def __init__(self, annotations): self.doc = make_dict(definitions=make_dict()) self.definitions = self.doc["definitions"] self.ns = NameStore() self.cw = CommentWriter() self.annotations = annotations or {} # Dict[string, Dict]
def _on_object(info): r = make_dict(type="object") r["properties"] = make_dict() r["required"] = [] for name, child in info["children"].items(): r["properties"][name] = _on_schema(child) if child.get("freq2") or (child["freq"] == info["freq"]): r["required"].append(name) if not r["required"]: r.pop("required") if not r["properties"]: r.pop("properties") return r
def run(*, src: str) -> None: resolver = get_resolver(src) with Migration(resolver).migrate_dryrun_and_diff() as m: item_map = m.item_map for k, item in list(item_map.items()): if k == "definitions/person": item.data["properties"] = ChainMap(make_dict(), item.data["properties"]) item.data["properties"]["value"] = {"type": "integer"} if k == "definitions/name": item.data = ChainMap(make_dict(), item.data) item.resolver.assign_by_json_pointer(item.localref, item.data) item.data["description"] = "name of something"
def _scan(self, doc, *, resolver, seen: dict): if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc, _ = self._scan( original, resolver=self.accessor.resolver, seen=seen ) return new_doc, self.accessor.pop_stack() else: for path, sd in self.ref_walking.iterate(doc): try: uid = id(sd) if uid in seen: continue seen[uid] = sd new_sd, sresolver = self._scan(sd, resolver=resolver, seen=seen) if resolver.filename != sresolver.filename: container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except (KeyError, FileNotFoundError) as e: self.errors.append( ReferenceError(e, store=self.store, path=path[:], data=sd) ) except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append( ParseError(e, store=self.store, path=path[:], data=sd) ) return doc, resolver
def emit(self, resolver, doc, *, conflicted): # side effect d = make_dict() for path, sd in self.ref_walking.iterate(doc): self.replace_ref(resolver, sd) d = deepmerge(d, doc) for name, item in self.item_map.items(): if name == "": continue data = item.data # replace: <file.yaml>#/<ref> -> #/<ref> for path, sd in self.ref_walking.iterate(data): if not sd["$ref"].startswith("#/"): self.replace_ref(item.resolver, sd) if sd["$ref"] in conflicted: self.replace_ref(item.resolver, sd) self.raw_accessor.assign(d, name.split("/"), data) # adhoc paths support will_removes = set() paths = d.get("paths") or {} for path, sub in list(paths.items()): if "$ref" in sub and sub["$ref"].startswith("#/"): related_path = tuple(sub["$ref"][2:].split("/")) paths[path] = self.raw_accessor.access(d, related_path).copy() will_removes.add(related_path) for related_path in will_removes: self.raw_accessor.maybe_remove(d, related_path) return d
def _show_on_lifezero(d, *, path): if hasattr(d, "keys"): rep = make_dict() for k, v in d.items(): if hasattr(v, "keys"): rep[k] = f"{_describe_type(v)}@{len(v)}" elif isinstance(v, (list, tuple)): rep[k] = f"{_describe_type(v)}@{len(v)}" else: rep[k] = f"{_describe_type(v)}" sig = json.dumps(rep, sort_keys=True, default=str) if sig in sigmap: return sigmap[sig] sigmap[sig] = path_to_json_pointer(path) return rep elif isinstance(d, (list, tuple)): seen = {} for x in d: rep = _show(x, life=0, path=path) sig = json.dumps(rep, sort_keys=True, default=str) if sig in seen: continue seen[sig] = rep return {"$len": len(d), "$cases": list(seen.values())} else: return _describe_type(d)
def deepmerge(*ds, override=False, method="addtoset"): """deepmerge: methods in {METHODS!r}""".format(METHODS=METHODS) if len(ds) == 0: return make_dict() if override: warnings.warn( "override option is deprecated, will be removed, near future", category=DeprecationWarning, ) merge = _deepmerge_replace elif method == "addtoset": merge = partial(_deepmerge_extend, dedup=True) elif method == "append": merge = partial(_deepmerge_extend, dedup=False) elif method == "merge": merge = _deepmerge_merge elif method == "replace": merge = _deepmerge_replace else: raise ValueError( "unavailable method not in {METHODS!r}".format(METHODS=METHODS)) left = ds[0].__class__() for right in ds: if not right: continue left = merge(left, right) return left
def _show(d, *, path, life, showzero_callable=showzero_callable or _show_on_lifezero): if life == 0: # -1 is full expand return showzero_callable(d, path=path) if hasattr(d, "keys"): rep = make_dict() for k, v in d.items(): path.append(k) if hasattr(v, "__len__") and len(v) == 0: rep[k] = f"{_describe_type(v)}@{len(v)}" elif hasattr(v, "keys"): rep[k] = _show(v, life=life - 1, path=path) elif isinstance(v, (list, tuple)): rep[k] = _show(v, life=life - 1, path=path) else: rep[k] = f"{_describe_type(v)}" path.pop() sig = json.dumps(rep, sort_keys=True, default=str) if sig in sigmap: return sigmap[sig] sigmap[sig] = path_to_json_pointer(path) return rep elif isinstance(d, (list, tuple)): seen = {} members = [] path.append("[]") csigmap = {} for x in d: rep = _show(x, life=life - 1, path=path) sig = json.dumps(rep, sort_keys=True, default=str) if sig in seen: members.append(csigmap[sig]) continue rep = copy.deepcopy(rep) for ks, sd in walker.walk(rep): sd.pop(ks[-1]) seen[sig] = rep csigmap[ sig] = f"{path_to_json_pointer(path)}/$cases/{len(csigmap)}" members.append(csigmap[sig]) rep = {"$len": len(d)} if seen: rep["$cases"] = list(seen.values()) if members: rep["$members"] = members path.pop() sig = json.dumps(rep, sort_keys=True, default=str) if sig in sigmap: return sigmap[sig] sigmap[sig] = path_to_json_pointer(path) return rep else: return _describe_type(d)
def make_info(self): return { "freq": 0, "freq2": 0, "type": "any", "children": make_dict(), "values": [], }
def _on_array(info): r = make_dict(type="array") r["items"] = _on_schema(info, from_array=True) # xxx: if r["items"].get("type", "any") == "any": r.pop("items") return r
def _on_primitive(info): r = make_dict(type=info["type"]) if "format" in info: r["format"] = info["format"] if info["values"]: r["example"] = info["values"][0] if info.get("type2") == "null": r["nullable"] = True return r
def make_array_schema(self, info, parent): self.cw.write(info["name"] + "[]", info, parent=parent) d = make_dict(type="array") d["items"] = self.make_schema(info, parent=info, fromarray=True) schema_name = self.resolve_name(info, suffix="[]") self.definitions[schema_name] = d if info["ref"] + "[]" in self.annotations: d.update(self.annotations[info["ref"] + "[]"]) d.pop("name", None) return {"$ref": "#/definitions/{name}".format(name=schema_name)}
def __init__(self, resolver, *, make_dict=make_dict, dump_options=None, transform=None): self.resolver = resolver self.item_map = make_dict() self.make_dict = make_dict self.dump_options = dump_options or {} self._transform = transform
def transform(doc): heavy_defs = ["definitions", "schemas", "responses", "parameters", "paths"] r = make_dict() for k, v in doc.items(): if k in heavy_defs: continue r[k] = v for k in heavy_defs: if k in doc: r[k] = doc[k] return r
def make_primitive_schema(self, info): d = make_dict(type=info["type"]) if "format" in info: d["format"] = info["format"] if info["values"]: d["example"] = info["values"][0] if info.get("type2") == "null": d["x-nullable"] = True if info["ref"] in self.annotations: d.update(self.annotations[info["ref"]]) d.pop("name", None) return d
def run(*, src: str) -> None: resolver = get_resolver(src) accessor = CachedItemAccessor(resolver) item_map = make_dict() scanner = Scanner(accessor, item_map, strict=True) scanner.scan(resolver.doc) resolvers = set() for k, item in list(item_map.items()): if k == "definitions/person": item.data["properties"] = ChainMap(make_dict(), item.data["properties"]) item.data["properties"]["value"] = {"type": "integer"} if k == "definitions/name": item.data = ChainMap(make_dict(), item.data) item.resolver.assign_by_json_pointer(item.localref, item.data) item.data["description"] = "name of something" resolvers.add(item.resolver) for r in resolvers: print(os.path.relpath(r.name, start=os.getcwd())) for line in diff(before_data(r.doc), after_data(r.doc)): print(line)
def transform(doc, *, sort_keys=False): """reorder""" heavy_defs = ["definitions", "schemas", "responses", "parameters", "paths"] r = make_dict() for k, v in doc.items(): if k in heavy_defs: continue r[k] = v for k in heavy_defs: if k in doc: r[k] = doc[k] if sort_keys: r = str_dict(r) # side effect return r
def make_object_schema(self, info, parent, fromarray=False): if not fromarray: self.cw.write(info["name"], info, parent=parent) d = make_dict(type="object") d["properties"] = make_dict() props = d["properties"] for name, value in info["children"].items(): props[name] = self.make_schema(value, parent=info) required = [ name for name, f in info["children"].items() if (f.get("freq2") or f["freq"]) == info["freq"] ] if required: d["required"] = required if info.get("type2") == "null": d["x-nullable"] = True schema_name = self.resolve_name(info, fromarray=fromarray) self.definitions[schema_name] = d if info["ref"] in self.annotations: d.update(self.annotations[info["ref"]]) d.pop("name", None) return {"$ref": "#/definitions/{name}".format(name=schema_name)}
def shape( *, files, input_format, output_format, squash, skiplist, separator, with_type, with_example, full ): """shape""" from dictknife import shape dataset = [] for f in files: with _open(f) as rf: d = loading.load(rf, format=input_format) if squash: dataset.extend(d) else: dataset.append(d) rows = shape(dataset, squash=True, skiplist=skiplist, separator=separator) r = [] for row in rows: d = make_dict() d["path"] = row.path if with_type: typenames = [t.__name__ for t in row.type] d["type"] = typenames[0] if len(typenames) == 1 else typenames if with_example: if full: d["example"] = row.example elif not any(t in (list, dict) for t in row.type): d["example"] = row.example elif output_format in ("csv", "tsv"): d["example"] = "" # xxx r.append(d) if output_format is None: for d in r: print(*d.values()) else: loading.dumpfile(r, format=output_format)
def cat(*, files, dst, format, input_format, output_format, sort_keys, encoding=None, errors=None, size=None, slurp=False, extra=None, merge_method="addtoset"): from dictknife import deepmerge input_format = input_format or format d = make_dict() with contextlib.ExitStack() as s: for f in files: logger.debug("merge: %s", f) opener = loading.get_opener(filename=f, format=input_format, default=_open) rf = s.enter_context(opener(f, encoding=encoding, errors=errors)) if slurp: sd = (loading.loads(line, format=input_format) for line in rf) else: sd = loading.load(rf, format=input_format, errors=errors) if size is not None: sd = itertools.islice(sd, size) if hasattr(sd, "keys"): d = deepmerge(d, sd, method=merge_method) elif len(files) == 1: d = sd else: if not isinstance(d, (list, tuple)): d = [d] if d else [] d = deepmerge(d, sd, method=merge_method) loading.dumpfile(d, dst, format=output_format or format, sort_keys=sort_keys, extra=extra)
def emit_environ(structure, make_dict, parse): if hasattr(structure, "keys"): d = make_dict() for k, v in structure.items(): name, fn = parse(v) emitted = emit_environ(name, make_dict=make_dict, parse=parse) if emitted is None: continue if fn is not None: emitted = fn(emitted) d[k] = emitted return d elif isinstance(structure, (list, tuple)): return [ emit_environ(x, make_dict=make_dict, parse=parse) for x in structure ] else: return os.environ.get(structure)
def load(fp, *, loader=None, errors=None, make_dict=make_dict, null_value="null", **kwargs): keys = None while keys is None: line = next(fp) if "|" in line: keys = [tok.strip() for tok in line.strip("|\n").split("|")] maybe_nums = None while maybe_nums is None: line = next(fp) if "|" in line: maybe_nums = [ tok.rstrip().endswith(":") and not tok.lstrip().startswith(":") for tok in line.strip("|\n").split("|") ] for line in fp: if "|" not in line: continue row = make_dict() for name, maybe_num, tok in zip(keys, maybe_nums, line.strip("|\n").split("|")): val = tok.strip() if not val: continue elif val == null_value: row[name] = None elif maybe_num: try: if "." in val: row[name] = float(val) else: row[name] = int(val) except ValueError: row[name] = val else: row[name] = val yield row
def expand(self, doc=None, resolver=None, ctx=None): doc = doc or self.resolver.doc resolver = resolver or self.resolver if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc = self.expand(original, resolver=self.accessor.resolver, ctx=ctx) self.accessor.pop_stack() return new_doc else: for path, sd in self.ref_walking.iterate(doc): new_sd = self.expand(sd, resolver=resolver, ctx=ctx) container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) return doc
def __next__(self): if self.line_num == 0: # Used only for its side effect. self.fieldnames row = next(self.reader) self.line_num = self.reader.line_num # unlike the basic reader, we prefer not to return blanks, # because we will typically wind up with a dict full of None # values while row == []: row = next(self.reader) d = make_dict(zip(self.fieldnames, row)) lf = len(self.fieldnames) lr = len(row) if lf < lr: d[self.restkey] = row[lf:] elif lf > lr: for key in self.fieldnames[lr:]: d[key] = self.restval return d
def scan(self, doc=None, resolver=None, ctx=None): resolver = resolver or self.resolver try: doc = doc or resolver.doc except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append(ParseError(e, store=self.store)) if doc is None: doc = {} if "$ref" in doc: original = self.accessor.access(doc["$ref"]) new_doc = self.scan(original, resolver=self.accessor.resolver, ctx=ctx) self.accessor.pop_stack() return new_doc else: for path, sd in self.ref_walking.iterate(doc): try: new_sd = self.scan(sd, resolver=resolver, ctx=ctx) container = self.accessing.access(doc, path[:-1]) if not hasattr(container, "parents"): container = ChainMap(make_dict(), container) container.update(new_sd) self.accessing.assign(doc, path[:-1], container) except (KeyError, FileNotFoundError) as e: self.errors.append( ReferenceError(e, store=self.store, path=path[:], data=sd)) except MarkedYAMLError as e: if e.problem_mark is not None: self.errors.append( ParseError(e, store=self.store, path=path[:], data=sd)) return doc
def select(self, ref, *, doc=None, a=Accessor(), r=None): ref_wrap = ref if self.wrap else None doc = doc or self.resolver.doc r = r or self.r if r is None: r = make_dict() # too many? if "@" in ref: ref, ref_wrap = ref.split("@", 1) if ref: accessed = access_by_json_pointer(doc, ref, guess=True) else: accessed = doc if ref_wrap is None: return accessed wrap_path = json_pointer_to_path(ref_wrap) a.assign(r, wrap_path, accessed) return r
def select( *, src: str, dst: str, refs, unwrap, wrap, input_format: str, output_format: str, format: str, ): from dictknife.jsonknife import Expander from dictknife.jsonknife.accessor import assign_by_json_pointer from dictknife.jsonknife import get_resolver input_format = input_format or format resolver = get_resolver(src) expander = Expander(resolver) if unwrap and not refs: refs = [] refs.append(unwrap) if not refs: d = expander.expand() else: d = make_dict() for ref in refs: ref_wrap = wrap if "@" in ref: ref, ref_wrap = ref.split("@", 1) extracted = expander.expand_subpart(expander.access(ref)) if ref_wrap: assign_by_json_pointer(d, ref_wrap, extracted) else: d = deepmerge(d, extracted) loading.dumpfile(d, dst, format=output_format or format)
def _extract(state, d, *, path=None): path = path or [] typ = d.get("type") if "properties" in d or typ == "object": return _extract_object(state, d.get("properties") or {}, r=make_dict(), path=path) elif typ == "array": return _extract_array(state, d["items"], r=[], path=path) elif "example" in d: if isinstance(d["example"], (list, tuple)) and d["type"] != "array": size = min(state.limit, len(d["example"])) state.max_examples = max(state.max_examples, size) return d["example"][min(state.i, len(d["example"]) - 1)] return d["example"] elif "enum" in d: size = min(state.limit, len(d["enum"])) state.max_examples = max(state.max_examples, size) return d["enum"][min(state.i, len(d["enum"]) - 1)] elif "default" in d: return d["default"] else: return "<>"
def __init__(self, resolver): self.resolver = resolver self.accessor = CachedItemAccessor(resolver) self.item_map = make_dict()
def transform( d, *, default_content_type="application/json", is_specific_header=is_specific_header, get_value=get_value, with_response_type=True, with_request_type=True, with_cookies=True, include_all=False, ): r = make_dict() a = Accessor() for path, methods in d.items(): for method, entries in methods.items(): d = {"description": ""} seen_parameters = defaultdict(set) request_bodies: t.List[dict] = [] response_bodies_dict: t.Dict[t.Tuple[int, str], dict] = defaultdict(list) for e in entries: # request # params :: path,query,header,cookie parameters = [] for param_type, k, enabled in [ ("query", "queryString", True), ("header", "headers", True), ("cookie", "cookies", with_cookies), ]: if not enabled: continue seen = seen_parameters[k] for h in e["request"][k]: if h["name"] in seen: continue seen.add(h["name"]) if include_all or is_specific_header(h["name"], h["value"]): parameters.append( { "name": h["name"], "in": param_type, "example": get_value( h["name"], h["value"] ), # masking? } ) if parameters: d["parameters"] = parameters if e["request"].get("postData"): post_data = e["request"]["postData"] content_type = post_data["mimeType"].split(";", 1)[0] if content_type.endswith("/json") and with_request_type: request_bodies.append( loading.loads(post_data["text"], format="json") ) # response status = e["response"]["status"] if status == 304: status = 200 # not modified -> ok content_type = e["response"]["content"].get("mimeType") if content_type is None: for h in e["response"]["headers"]: if h["name"].lower() == "content-type": content_type = h["value"] break else: content_type = default_content_type # "application/json; charset=utf-8" -> "application/json" content_type = content_type.split(";", 1)[0] schema = {} if content_type.startswith("text/"): a.assign(schema, ["type"], "string") elif content_type.endswith("/json") and with_response_type: response_bodies_dict[(status, content_type)].append( loading.loads(e["response"]["content"]["text"], format="json") ) a.assign( d, ["responses", status], { "description": e["response"]["statusText"], "content": {content_type: {"schema": schema}}, }, ) if request_bodies: detector = schemalib.Detector() info = None for body in request_bodies: info = detector.detect(body, name="") a.assign(d, ["requestBody"], schemalib.makeschema_from_info(info)) if response_bodies_dict: for (status, content_type), bodies in response_bodies_dict.items(): detector = schemalib.Detector() info = None for body in bodies: info = detector.detect(body, name="") a.assign( d, ["responses", status, "content", content_type, "schema"], schemalib.makeschema_from_info(info), ) a.assign(r, ["paths", path, method.lower()], d) return r
def __init__(self, resolver, *, make_dict=make_dict): self.resolver = resolver self.item_map = make_dict() self.make_dict = make_dict