def data(self): data = deepmerge( *[loading.loadfile(d) for d in self.data_path_list], override=True ) if self.format is not None: data = deepmerge( data, loading.load(sys.stdin, format=self.format), override=True ) return data
def merge( *, files: list, dst: str, style: str, # flavor?, strategy? strict: bool = False, wrap: str = None, wrap_section: str = "definitions" ): """merge files""" from dictknife.langhelpers import make_dict, as_jsonpointer from dictknife import deepmerge if style == "ref": dstdir = dst and os.path.dirname(dst) r = make_dict() seen = {} for src in files: d = loading.loadfile(src) for ns, sd in d.items(): for name in sd: if ns not in r: r[ns] = make_dict() seen[ns] = make_dict() if strict and name in r[ns]: raise RuntimeError( "{name} is already existed, (where={where} and {where2})".format( name=name, where=seen[ns][name], where2=src ) ) if dst is None: where = "" else: where = os.path.relpath(src, start=dstdir) r[ns][name] = { "$ref": "{where}#/{ns}/{name}".format( where=where, ns=ns, name=as_jsonpointer(name) ) } seen[ns][name] = src elif style == "whole": # TODO: strict support? data = [loading.loadfile(src) for src in files] r = deepmerge(*data, override=True) else: raise RuntimeError("invalid style: {}".format(style)) if wrap is not None: wd = make_dict() wd["type"] = "object" wd["properties"] = make_dict() for name in r.get(wrap_section) or {}: wd["properties"][name] = { "$ref": "#/{wrap_section}/{name}".format( wrap_section=wrap_section, name=name ) } r[wrap_section][wrap] = wd loading.dumpfile(r, dst)
def transform(*, src: str, dst: str, config: str, config_file: str, code: str, functions: str, input_format: str, output_format: str, format: str, sort_keys: str): """transform dict""" from magicalimport import import_symbol from dictknife import deepmerge if code is not None: transform = eval(code) elif functions: def transform(d): for fn in functions: if "." not in fn: fn = "dictknife.transform:{}".format(fn) d = import_symbol(fn)(d) return d else: transform = lambda x: x # NOQA input_format = input_format or format kwargs = loading.loads(config, format=input_format) if config_file: with open(config_file) as rf: kwargs = deepmerge(kwargs, loading.load(rf, format=input_format)) data = loading.loadfile(src, input_format) result = transform(data, **kwargs) loading.dumpfile(result, dst, format=output_format or format, sort_keys=sort_keys)
def emit(self, resolver, doc, *, conflicted): # side effect d = make_dict() for path, sd in self.ref_walking.iterate(doc): self.replace_ref(resolver, sd) d = deepmerge(d, doc) for name, item in self.item_map.items(): if name == "": continue data = item.data # replace: <file.yaml>#/<ref> -> #/<ref> for path, sd in self.ref_walking.iterate(data): if not sd["$ref"].startswith("#/"): self.replace_ref(item.resolver, sd) if sd["$ref"] in conflicted: self.replace_ref(item.resolver, sd) self.raw_accessor.assign(d, name.split("/"), data) # adhoc paths support will_removes = set() paths = d.get("paths") or {} for path, sub in list(paths.items()): if "$ref" in sub and sub["$ref"].startswith("#/"): related_path = tuple(sub["$ref"][2:].split("/")) paths[path] = self.raw_accessor.access(d, related_path).copy() will_removes.add(related_path) for related_path in will_removes: self.raw_accessor.maybe_remove(d, related_path) return d
def _build_additionals(modules, *, import_string) -> dict: additionals = {} for name in modules: logger.info("activate additional module %s", name) m = import_string(name) # xxx: use magicalimport.import_module()? additionals = deepmerge(additionals, collect_marked_items(m)) return additionals
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument("--data", action="append", help="support yaml, json, toml", default=[]) parser.add_argument("template") parser.add_argument("--input-format", default=None) parser.add_argument("--output-format", default="raw") parser.add_argument("--dst", default=None) args = parser.parse_args() loading.setup() data = deepmerge(*[loadfile(d) for d in args.data], override=True) if args.input_format is not None: data = deepmerge(data, loading.load(sys.stdin, format=args.input_format), override=True) result = run(args.template, data) loading.dumpfile(result, args.dst, format=args.output_format)
def run(*, config: str) -> None: c = loading.loadfile(config) overwrite_file = os.environ.get("OVERWRITE_CONFIG") if overwrite_file is not None: c2 = loading.loadfile(overwrite_file) c = deepmerge(c, c2, method="replace") loading.dumpfile(c)
def inherit(ref, context, **kwargs): data = context.loader.data if ref.startswith("#/"): parent = access_by_json_pointer(data, ref[1:]) parent = context.evaluator.eval(context, parent) else: parent = unquote(load(ref, context)) return quote(deepmerge(parent, kwargs, override=True))
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument("--data", action="append", help="support yaml, json, toml", default=[]) parser.add_argument("template") parser.add_argument("--input-format", default=None) parser.add_argument("--output-format", default="raw") parser.add_argument("--dst", default=None) args = parser.parse_args() loading.setup() data = deepmerge(*[loading.loadfile(d) for d in args.data], override=True) if args.input_format is not None: data = deepmerge(data, loading.load(sys.stdin, format=args.input_format), override=True) result = run(args.template, data) loading.dumpfile(result, args.dst, format=args.output_format)
def cat(*, files, dst, format, input_format, output_format, sort_keys, encoding=None, errors=None, size=None, slurp=False, extra=None, merge_method="addtoset"): from dictknife import deepmerge input_format = input_format or format d = make_dict() with contextlib.ExitStack() as s: for f in files: logger.debug("merge: %s", f) opener = loading.get_opener(filename=f, format=input_format, default=_open) rf = s.enter_context(opener(f, encoding=encoding, errors=errors)) if slurp: sd = (loading.loads(line, format=input_format) for line in rf) else: sd = loading.load(rf, format=input_format, errors=errors) if size is not None: sd = itertools.islice(sd, size) if hasattr(sd, "keys"): d = deepmerge(d, sd, method=merge_method) elif len(files) == 1: d = sd else: if not isinstance(d, (list, tuple)): d = [d] if d else [] d = deepmerge(d, sd, method=merge_method) loading.dumpfile(d, dst, format=output_format or format, sort_keys=sort_keys, extra=extra)
def resolve_allof_definition(self, fulldata, d): ref_list = [] r = OrderedDict() for subdef in d["allOf"]: if self.has_ref(subdef): ref_list.append(self.resolve_ref_definition(fulldata, subdef)) else: r = dictknife.deepmerge(r, subdef) return ref_list, r
def additionals(self): d = {} for path in self.additional_path_list: try: m = import_module(path, cwd=True) except ImportError: m = import_module("kamidana.additionals.{}".format(path), cwd=True) d = deepmerge(d, collect_marked_items(m)) return d
def concat(ds, override=False, sep="\n"): """ $concat: - name: foo - age: 10 """ if not ds: return {} elif hasattr(ds[0], "keys"): return quote(deepmerge(*ds, override=override)) else: return sep.join(ds)
def _load_data(self, name_or_data, *, cache): if name_or_data is None: return {} elif isinstance(name_or_data, (list, tuple)): return deepmerge( *[self._load_data(d, cache=cache) for d in name_or_data]) elif hasattr(name_or_data, "get"): return name_or_data else: r = cache.get(name_or_data) if r is None: r = cache[name_or_data] = loading.loadfile(name_or_data) return r
def select( *, src: str, dst: str, refs, unwrap, wrap, input_format: str, output_format: str, format: str, ): from dictknife.jsonknife import Expander from dictknife.jsonknife.accessor import assign_by_json_pointer from dictknife.jsonknife import get_resolver input_format = input_format or format resolver = get_resolver(src) expander = Expander(resolver) if unwrap and not refs: refs = [] refs.append(unwrap) if not refs: d = expander.expand() else: d = make_dict() for ref in refs: ref_wrap = wrap if "@" in ref: ref, ref_wrap = ref.split("@", 1) extracted = expander.expand_subpart(expander.access(ref)) if ref_wrap: assign_by_json_pointer(d, ref_wrap, extracted) else: d = deepmerge(d, extracted) loading.dumpfile(d, dst, format=output_format or format)
def load(self, batch_file): commands = loading.loadfile(batch_file) if not isinstance(commands, (list, tuple)): commands = [commands] core_data = self.loader.data cache = {} r = [] for cmd in commands: # require: template, dst for name in ["template", "dst"]: if name not in cmd: raise RuntimeError( "{} is missing. this is required field. (passed command={})" .format(name, json.dumps(cmd, ensure_ascii=False))) data = self._load_data(cmd.get("data"), cache=cache) tname = cmd["template"] t = cache.get(tname) if t is None: t = cache[tname] = self.environment.get_or_select_template( tname) r.append((t, cmd, deepmerge(data, core_data))) return r
from dictknife import deepmerge d = { "xs": { "a": { "name": "a", "vs": [1, 2, 3] } }, } d2 = { "xs": { "a": { "name": "a'", "vs": [10] } }, } print(deepmerge(d, d2, make_dict=dict)) # {'xs': {'a': {'vs': [1, 2, 3, 10], 'name': "a'"}}} print(deepmerge(d, d2, override=True, make_dict=dict)) # {'xs': {'a': {'vs': [10], 'name': "a'"}}}
from dictknife import deepmerge d = {"db": {"name": "foo", "tables": [{"name": "b"}]}} d2 = {"db": {"name": "foo@", "tables": [{"name": "a"}]}} print(deepmerge(d, d2))
from dictknife import deepmerge d = {"db": {"name": "foo", "tables": [{"name": "b"}]}} d2 = {"db": {"name": "foo@", "tables": [{"name": "a"}]}} print(deepmerge(d, d2, method="replace"))
def cmd_main( ctx: click.Context, user_configs: typing.Iterator[typing.TextIO], root_stream_log_level: int, logfiles: typing.Iterable[typing.Tuple[int, pathlib.Path]], ): """A CLI toolkit to generate and manupilate the ABC Treebank. \f :param ctx: The context argument that is used by Click. :param user_configs: List of streams of configuration files. Each file is opened beforehand and will be closed properly by Click. :param root_stream_log_level: The log level for the root logger. Specified in integer. :param logfiles: List of additional log handlers. Each item consists of a tuple of a log level and an output path. """ ctx.ensure_object(dict) # ==================== # Configure logging # ==================== # Adjust the root stream handler logger_root = logging.getLogger() logger_root.setLevel(root_stream_log_level) logging.info( f"The log level of the root logger is set to {logger_root.level}") # Add file handlers to the root for level, hpath in logfiles: hd = logging.FileHandler(hpath) hd.setLevel(level) logger_root.addHandler(hd) # === END FOR level, hpath === logger_root.info( f"The handler(s) of the root logger is/are: {logging.root.handlers}") # ==================== # Build config # ==================== from dictknife import deepmerge import ruamel.yaml yaml = ruamel.yaml.YAML() import xdg import os CONFIG: typing.Dict[str, typing.Any] = CONF.CONF_DEFAULT path_config_user: pathlib.Path = xdg.xdg_config_home() / "ABCTreebank.yaml" if os.path.exists(path_config_user): with open(path_config_user, "r") as cu: CONFIG = deepmerge(CONFIG, cu, method="merge") # === END WITH cu === # === END IF === for h_cf in user_configs: CONFIG = deepmerge(CONFIG, yaml.load(h_cf), method="merge") # === END IF === ctx.obj["CONFIG"] = CONFIG
d0 = { "name": "foo", "object": { "x": 1, "z": 1, }, "children": [10], "a": { "b": { "x": "y" } }, } d1 = { "name": "bar", "object": { "y": 2, "z": 3, }, "children": [1, 2, 3], "a": { "b": { "c": "d", "x": "z", } }, } pp(deepmerge(d0, d1, override=True))
def context_factory(self, *args, **kwargs): context = Context(*args, **kwargs) context.assign("data", deepmerge(*self.data, override=False)) # xxx return context
def write_schema(self, c, d, clsname, definition, force=False, meta_writer=None, many=False): if not force and clsname in self.arrived: return self.arrived.add(clsname) base_classes = [self.schema_class] if self.resolver.has_ref(definition): ref_name, ref_definition = self.resolver.resolve_ref_definition( d, definition) if ref_name is None: raise CodegenError("$ref %r is not found", definition["$ref"]) elif "items" in ref_definition: # work around many = True items = ref_definition["items"] if self.resolver.has_ref(items): _, items = self.resolver.resolve_ref_definition( d, ref_definition["items"]) if not self.resolver.has_schema(d, items): return self.write_primitive_schema(c, d, clsname, ref_definition, many=many) else: self.write_schema(c, d, ref_name, items) base_classes = [ref_name] else: if not self.resolver.has_schema(d, ref_definition): return self.write_primitive_schema(c, d, clsname, ref_definition, many=many) self.write_schema(c, d, ref_name, ref_definition) base_classes = [ref_name] elif self.resolver.has_allof(definition): ref_list, ref_definition = self.resolver.resolve_allof_definition( d, definition) definition = deepmerge(ref_definition, definition) if ref_list: base_classes = [] for ref_name, ref_definition in ref_list: if ref_name is None: raise CodegenError("$ref %r is not found", ref_definition) # xxx else: self.write_schema(c, d, ref_name, ref_definition) base_classes.append(ref_name) # supporting additional properties if "additionalProperties" in definition and base_classes[ 0] == self.schema_class: c.from_("swagger_marshmallow_codegen.schema", "AdditionalPropertiesSchema") base_classes[0] = "AdditionalPropertiesSchema" if "properties" not in definition and ("object" != definition.get( "type", "object") and "items" not in definition): return self.write_primitive_schema(c, d, clsname, definition, many=many) if "items" in definition: many = True if not self.resolver.has_ref(definition["items"]): return self.write_primitive_schema(c, d, clsname, definition, many=many) else: ref_name, ref_definition = self.resolver.resolve_ref_definition( d, definition["items"]) if ref_name is None: return self.write_primitive_schema(c, d, clsname, definition, many=many) else: self.write_schema(c, d, ref_name, ref_definition) base_classes = [ref_name] with c.m.class_(clsname, bases=base_classes): if "description" in definition: c.m.stmt('"""{}"""'.format(definition["description"])) if meta_writer is not None: meta_writer(c.m) if many or self.resolver.has_many(definition): with c.m.def_("__init__", "self", "*args", "**kwargs"): c.m.stmt("kwargs['many'] = True") c.m.stmt("super().__init__(*args, **kwargs)") opts = defaultdict(OrderedDict) self.accessor.update_options_pre_properties(definition, opts) properties = self.accessor.properties(definition) need_pass_statement = False if not properties and not many: need_pass_statement = True else: for name, field in properties.items(): name = str(name) if self.resolver.has_many(field): self.write_field_many(c, d, clsname, definition, name, field, opts[name]) else: self.write_field_one(c, d, clsname, definition, name, field, opts[name]) # supporting additional properties subdef = definition.get("additionalProperties") if subdef and hasattr(subdef, "keys"): need_pass_statement = False c.m.sep() subdef = definition["additionalProperties"] with c.m.class_("Meta"): if self.resolver.has_ref(subdef): ref_name, _ = self.resolver.resolve_ref_definition( d, subdef) if ref_name is None: raise CodegenError("$ref %r is not found", subdef["$ref"]) self.write_field_one(c, d, ref_name, {}, "additional_field", subdef, OrderedDict()) else: self.write_field_one(c, d, "", subdef, "additional_field", subdef, OrderedDict()) if need_pass_statement: c.m.stmt("pass")
def _callFUT(self, *args, **kwargs): from dictknife import deepmerge return deepmerge(*args, **kwargs)
d0 = { "name": "foo", "object": { "x": 1, "z": 1, }, "children": [10], "a": { "b": { "x": "y" } }, } d1 = { "name": "bar", "object": { "y": 2, "z": 3, }, "children": [1, 2, 3], "a": { "b": { "c": "d", "x": "z", } }, } pp(deepmerge(d0, d1))
def expand(self, ctx: Context, data: dict, *, minimize=True) -> dict: pool = self.pool seen = self.seen genid = self.genid accessor = self.accessor ref_walker = self.ref_walker xxx_of_walker = self.xxx_of_walker def _on_newdata(resolver, data): for path, sd in xxx_of_walker.walk(data): cases = sd[path[-1]] for i, case in enumerate(cases): if "$ref" in case: continue # todo: optimization kid = genid() assigned.add(kid) pool[kid] = case cases[i] = {"$ref": f"#/definitions/{kid}"} seen[(resolver, f"definitions/{kid}")] = kid # xxx return data r = copy.deepcopy(data) # todo: merge code with ..context:Context.get_uid() # todo: drop description? q = [] q.append((ctx.resolver, r)) assigned = set() while q: resolver, d = q.pop() for path, sd in ref_walker.walk(d): history = [] kid = genid() k = (resolver, "/".join(map(str, path)).lstrip("#/")) if k in seen: assigned_kid = seen[k] assigned.add(assigned_kid) sd["$ref"] = f"#/definitions/{assigned_kid}" continue seen[k] = kid history.append(k) stack = [(resolver, sd["$ref"])] found = False while True: resolver, ref = stack[-1] k = (resolver, ref.lstrip("#/")) if k in seen: assigned_kid = seen[k] assigned.add(assigned_kid) sd["$ref"] = f"#/definitions/{assigned_kid}" for k in history: seen[k] = assigned_kid break seen[k] = kid history.append(k) sresolver, sref = resolver.resolve(ref) sd = sresolver.access_by_json_pointer(sref) if not hasattr(sd, "get") or "$ref" not in sd: found = True stack.append((sresolver, sd)) break stack.append((sresolver, sd["$ref"])) if not found: continue sresolver, sd = stack[-1] new_sd = copy.deepcopy(sd) if minimize: for _, prop in list(self.description_walker.walk(new_sd)): prop.pop("description") pool[kid] = _on_newdata(sresolver, new_sd) assigned.add(kid) accessor.assign(r, path, f"#/definitions/{kid}") q.append((sresolver, new_sd)) r = _on_newdata(ctx.resolver, r) if not assigned: return r return deepmerge( r, {"definitions": {str(kid): pool[kid] for kid in sorted(assigned)}})