def data(self):
data = deepmerge(
*[loading.loadfile(d) for d in self.data_path_list], override=True
)
if self.format is not None:
data = deepmerge(
data, loading.load(sys.stdin, format=self.format), override=True
)
return data
def merge(
*,
files: list,
dst: str,
style: str, # flavor?, strategy?
strict: bool = False,
wrap: str = None,
wrap_section: str = "definitions"
):
"""merge files"""
from dictknife.langhelpers import make_dict, as_jsonpointer
from dictknife import deepmerge
if style == "ref":
dstdir = dst and os.path.dirname(dst)
r = make_dict()
seen = {}
for src in files:
d = loading.loadfile(src)
for ns, sd in d.items():
for name in sd:
if ns not in r:
r[ns] = make_dict()
seen[ns] = make_dict()
if strict and name in r[ns]:
raise RuntimeError(
"{name} is already existed, (where={where} and {where2})".format(
name=name, where=seen[ns][name], where2=src
)
)
if dst is None:
where = ""
else:
where = os.path.relpath(src, start=dstdir)
r[ns][name] = {
"$ref": "{where}#/{ns}/{name}".format(
where=where, ns=ns, name=as_jsonpointer(name)
)
}
seen[ns][name] = src
elif style == "whole":
# TODO: strict support?
data = [loading.loadfile(src) for src in files]
r = deepmerge(*data, override=True)
else:
raise RuntimeError("invalid style: {}".format(style))
if wrap is not None:
wd = make_dict()
wd["type"] = "object"
wd["properties"] = make_dict()
for name in r.get(wrap_section) or {}:
wd["properties"][name] = {
"$ref": "#/{wrap_section}/{name}".format(
wrap_section=wrap_section, name=name
)
}
r[wrap_section][wrap] = wd
loading.dumpfile(r, dst)
def transform(*, src: str, dst: str, config: str, config_file: str, code: str,
functions: str, input_format: str, output_format: str,
format: str, sort_keys: str):
"""transform dict"""
from magicalimport import import_symbol
from dictknife import deepmerge
if code is not None:
transform = eval(code)
elif functions:
def transform(d):
for fn in functions:
if "." not in fn:
fn = "dictknife.transform:{}".format(fn)
d = import_symbol(fn)(d)
return d
else:
transform = lambda x: x # NOQA
input_format = input_format or format
kwargs = loading.loads(config, format=input_format)
if config_file:
with open(config_file) as rf:
kwargs = deepmerge(kwargs, loading.load(rf, format=input_format))
data = loading.loadfile(src, input_format)
result = transform(data, **kwargs)
loading.dumpfile(result,
dst,
format=output_format or format,
sort_keys=sort_keys)
def emit(self, resolver, doc, *, conflicted):
# side effect
d = make_dict()
for path, sd in self.ref_walking.iterate(doc):
self.replace_ref(resolver, sd)
d = deepmerge(d, doc)
for name, item in self.item_map.items():
if name == "":
continue
data = item.data
# replace: <file.yaml>#/<ref> -> #/<ref>
for path, sd in self.ref_walking.iterate(data):
if not sd["$ref"].startswith("#/"):
self.replace_ref(item.resolver, sd)
if sd["$ref"] in conflicted:
self.replace_ref(item.resolver, sd)
self.raw_accessor.assign(d, name.split("/"), data)
# adhoc paths support
will_removes = set()
paths = d.get("paths") or {}
for path, sub in list(paths.items()):
if "$ref" in sub and sub["$ref"].startswith("#/"):
related_path = tuple(sub["$ref"][2:].split("/"))
paths[path] = self.raw_accessor.access(d, related_path).copy()
will_removes.add(related_path)
for related_path in will_removes:
self.raw_accessor.maybe_remove(d, related_path)
return d
def _build_additionals(modules, *, import_string) -> dict:
additionals = {}
for name in modules:
logger.info("activate additional module %s", name)
m = import_string(name) # xxx: use magicalimport.import_module()?
additionals = deepmerge(additionals, collect_marked_items(m))
return additionals
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--data", action="append", help="support yaml, json, toml", default=[])
parser.add_argument("template")
parser.add_argument("--input-format", default=None)
parser.add_argument("--output-format", default="raw")
parser.add_argument("--dst", default=None)
args = parser.parse_args()
loading.setup()
data = deepmerge(*[loadfile(d) for d in args.data], override=True)
if args.input_format is not None:
data = deepmerge(data, loading.load(sys.stdin, format=args.input_format), override=True)
result = run(args.template, data)
loading.dumpfile(result, args.dst, format=args.output_format)
def run(*, config: str) -> None:
c = loading.loadfile(config)
overwrite_file = os.environ.get("OVERWRITE_CONFIG")
if overwrite_file is not None:
c2 = loading.loadfile(overwrite_file)
c = deepmerge(c, c2, method="replace")
loading.dumpfile(c)
def inherit(ref, context, **kwargs):
data = context.loader.data
if ref.startswith("#/"):
parent = access_by_json_pointer(data, ref[1:])
parent = context.evaluator.eval(context, parent)
else:
parent = unquote(load(ref, context))
return quote(deepmerge(parent, kwargs, override=True))
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--data", action="append", help="support yaml, json, toml", default=[])
parser.add_argument("template")
parser.add_argument("--input-format", default=None)
parser.add_argument("--output-format", default="raw")
parser.add_argument("--dst", default=None)
args = parser.parse_args()
loading.setup()
data = deepmerge(*[loading.loadfile(d) for d in args.data], override=True)
if args.input_format is not None:
data = deepmerge(data, loading.load(sys.stdin, format=args.input_format), override=True)
result = run(args.template, data)
loading.dumpfile(result, args.dst, format=args.output_format)
def cat(*,
files,
dst,
format,
input_format,
output_format,
sort_keys,
encoding=None,
errors=None,
size=None,
slurp=False,
extra=None,
merge_method="addtoset"):
from dictknife import deepmerge
input_format = input_format or format
d = make_dict()
with contextlib.ExitStack() as s:
for f in files:
logger.debug("merge: %s", f)
opener = loading.get_opener(filename=f,
format=input_format,
default=_open)
rf = s.enter_context(opener(f, encoding=encoding, errors=errors))
if slurp:
sd = (loading.loads(line, format=input_format) for line in rf)
else:
sd = loading.load(rf, format=input_format, errors=errors)
if size is not None:
sd = itertools.islice(sd, size)
if hasattr(sd, "keys"):
d = deepmerge(d, sd, method=merge_method)
elif len(files) == 1:
d = sd
else:
if not isinstance(d, (list, tuple)):
d = [d] if d else []
d = deepmerge(d, sd, method=merge_method)
loading.dumpfile(d,
dst,
format=output_format or format,
sort_keys=sort_keys,
extra=extra)
def resolve_allof_definition(self, fulldata, d):
ref_list = []
r = OrderedDict()
for subdef in d["allOf"]:
if self.has_ref(subdef):
ref_list.append(self.resolve_ref_definition(fulldata, subdef))
else:
r = dictknife.deepmerge(r, subdef)
return ref_list, r
def additionals(self):
d = {}
for path in self.additional_path_list:
try:
m = import_module(path, cwd=True)
except ImportError:
m = import_module("kamidana.additionals.{}".format(path), cwd=True)
d = deepmerge(d, collect_marked_items(m))
return d
def concat(ds, override=False, sep="\n"):
"""
$concat:
- name: foo
- age: 10
"""
if not ds:
return {}
elif hasattr(ds[0], "keys"):
return quote(deepmerge(*ds, override=override))
else:
return sep.join(ds)
def _load_data(self, name_or_data, *, cache):
if name_or_data is None:
return {}
elif isinstance(name_or_data, (list, tuple)):
return deepmerge(
*[self._load_data(d, cache=cache) for d in name_or_data])
elif hasattr(name_or_data, "get"):
return name_or_data
else:
r = cache.get(name_or_data)
if r is None:
r = cache[name_or_data] = loading.loadfile(name_or_data)
return r
def select(
*,
src: str,
dst: str,
refs,
unwrap,
wrap,
input_format: str,
output_format: str,
format: str,
):
from dictknife.jsonknife import Expander
from dictknife.jsonknife.accessor import assign_by_json_pointer
from dictknife.jsonknife import get_resolver
input_format = input_format or format
resolver = get_resolver(src)
expander = Expander(resolver)
if unwrap and not refs:
refs = []
refs.append(unwrap)
if not refs:
d = expander.expand()
else:
d = make_dict()
for ref in refs:
ref_wrap = wrap
if "@" in ref:
ref, ref_wrap = ref.split("@", 1)
extracted = expander.expand_subpart(expander.access(ref))
if ref_wrap:
assign_by_json_pointer(d, ref_wrap, extracted)
else:
d = deepmerge(d, extracted)
loading.dumpfile(d, dst, format=output_format or format)
def load(self, batch_file):
commands = loading.loadfile(batch_file)
if not isinstance(commands, (list, tuple)):
commands = [commands]
core_data = self.loader.data
cache = {}
r = []
for cmd in commands:
# require: template, dst
for name in ["template", "dst"]:
if name not in cmd:
raise RuntimeError(
"{} is missing. this is required field. (passed command={})"
.format(name, json.dumps(cmd, ensure_ascii=False)))
data = self._load_data(cmd.get("data"), cache=cache)
tname = cmd["template"]
t = cache.get(tname)
if t is None:
t = cache[tname] = self.environment.get_or_select_template(
tname)
r.append((t, cmd, deepmerge(data, core_data)))
return r
from dictknife import deepmerge
d = {
"xs": {
"a": {
"name": "a",
"vs": [1, 2, 3]
}
},
}
d2 = {
"xs": {
"a": {
"name": "a'",
"vs": [10]
}
},
}
print(deepmerge(d, d2, make_dict=dict))
# {'xs': {'a': {'vs': [1, 2, 3, 10], 'name': "a'"}}}
print(deepmerge(d, d2, override=True, make_dict=dict))
# {'xs': {'a': {'vs': [10], 'name': "a'"}}}
from dictknife import deepmerge
d = {"db": {"name": "foo", "tables": [{"name": "b"}]}}
d2 = {"db": {"name": "foo@", "tables": [{"name": "a"}]}}
print(deepmerge(d, d2))
from dictknife import deepmerge
d = {"db": {"name": "foo", "tables": [{"name": "b"}]}}
d2 = {"db": {"name": "foo@", "tables": [{"name": "a"}]}}
print(deepmerge(d, d2, method="replace"))
def cmd_main(
ctx: click.Context,
user_configs: typing.Iterator[typing.TextIO],
root_stream_log_level: int,
logfiles: typing.Iterable[typing.Tuple[int, pathlib.Path]],
):
"""A CLI toolkit to generate and manupilate the ABC Treebank.
\f
:param ctx: The context argument that is used by Click.
:param user_configs:
List of streams of configuration files.
Each file is opened beforehand
and will be closed properly by Click.
:param root_stream_log_level:
The log level for the root logger. Specified in integer.
:param logfiles:
List of additional log handlers.
Each item consists of a tuple of a log level and an output path.
"""
ctx.ensure_object(dict)
# ====================
# Configure logging
# ====================
# Adjust the root stream handler
logger_root = logging.getLogger()
logger_root.setLevel(root_stream_log_level)
logging.info(
f"The log level of the root logger is set to {logger_root.level}")
# Add file handlers to the root
for level, hpath in logfiles:
hd = logging.FileHandler(hpath)
hd.setLevel(level)
logger_root.addHandler(hd)
# === END FOR level, hpath ===
logger_root.info(
f"The handler(s) of the root logger is/are: {logging.root.handlers}")
# ====================
# Build config
# ====================
from dictknife import deepmerge
import ruamel.yaml
yaml = ruamel.yaml.YAML()
import xdg
import os
CONFIG: typing.Dict[str, typing.Any] = CONF.CONF_DEFAULT
path_config_user: pathlib.Path = xdg.xdg_config_home() / "ABCTreebank.yaml"
if os.path.exists(path_config_user):
with open(path_config_user, "r") as cu:
CONFIG = deepmerge(CONFIG, cu, method="merge")
# === END WITH cu ===
# === END IF ===
for h_cf in user_configs:
CONFIG = deepmerge(CONFIG, yaml.load(h_cf), method="merge")
# === END IF ===
ctx.obj["CONFIG"] = CONFIG
d0 = {
"name": "foo",
"object": {
"x": 1,
"z": 1,
},
"children": [10],
"a": {
"b": {
"x": "y"
}
},
}
d1 = {
"name": "bar",
"object": {
"y": 2,
"z": 3,
},
"children": [1, 2, 3],
"a": {
"b": {
"c": "d",
"x": "z",
}
},
}
pp(deepmerge(d0, d1, override=True))
def context_factory(self, *args, **kwargs):
context = Context(*args, **kwargs)
context.assign("data", deepmerge(*self.data, override=False)) # xxx
return context
def write_schema(self,
c,
d,
clsname,
definition,
force=False,
meta_writer=None,
many=False):
if not force and clsname in self.arrived:
return
self.arrived.add(clsname)
base_classes = [self.schema_class]
if self.resolver.has_ref(definition):
ref_name, ref_definition = self.resolver.resolve_ref_definition(
d, definition)
if ref_name is None:
raise CodegenError("$ref %r is not found", definition["$ref"])
elif "items" in ref_definition:
# work around
many = True
items = ref_definition["items"]
if self.resolver.has_ref(items):
_, items = self.resolver.resolve_ref_definition(
d, ref_definition["items"])
if not self.resolver.has_schema(d, items):
return self.write_primitive_schema(c,
d,
clsname,
ref_definition,
many=many)
else:
self.write_schema(c, d, ref_name, items)
base_classes = [ref_name]
else:
if not self.resolver.has_schema(d, ref_definition):
return self.write_primitive_schema(c,
d,
clsname,
ref_definition,
many=many)
self.write_schema(c, d, ref_name, ref_definition)
base_classes = [ref_name]
elif self.resolver.has_allof(definition):
ref_list, ref_definition = self.resolver.resolve_allof_definition(
d, definition)
definition = deepmerge(ref_definition, definition)
if ref_list:
base_classes = []
for ref_name, ref_definition in ref_list:
if ref_name is None:
raise CodegenError("$ref %r is not found",
ref_definition) # xxx
else:
self.write_schema(c, d, ref_name, ref_definition)
base_classes.append(ref_name)
# supporting additional properties
if "additionalProperties" in definition and base_classes[
0] == self.schema_class:
c.from_("swagger_marshmallow_codegen.schema",
"AdditionalPropertiesSchema")
base_classes[0] = "AdditionalPropertiesSchema"
if "properties" not in definition and ("object" != definition.get(
"type", "object") and "items" not in definition):
return self.write_primitive_schema(c,
d,
clsname,
definition,
many=many)
if "items" in definition:
many = True
if not self.resolver.has_ref(definition["items"]):
return self.write_primitive_schema(c,
d,
clsname,
definition,
many=many)
else:
ref_name, ref_definition = self.resolver.resolve_ref_definition(
d, definition["items"])
if ref_name is None:
return self.write_primitive_schema(c,
d,
clsname,
definition,
many=many)
else:
self.write_schema(c, d, ref_name, ref_definition)
base_classes = [ref_name]
with c.m.class_(clsname, bases=base_classes):
if "description" in definition:
c.m.stmt('"""{}"""'.format(definition["description"]))
if meta_writer is not None:
meta_writer(c.m)
if many or self.resolver.has_many(definition):
with c.m.def_("__init__", "self", "*args", "**kwargs"):
c.m.stmt("kwargs['many'] = True")
c.m.stmt("super().__init__(*args, **kwargs)")
opts = defaultdict(OrderedDict)
self.accessor.update_options_pre_properties(definition, opts)
properties = self.accessor.properties(definition)
need_pass_statement = False
if not properties and not many:
need_pass_statement = True
else:
for name, field in properties.items():
name = str(name)
if self.resolver.has_many(field):
self.write_field_many(c, d, clsname, definition, name,
field, opts[name])
else:
self.write_field_one(c, d, clsname, definition, name,
field, opts[name])
# supporting additional properties
subdef = definition.get("additionalProperties")
if subdef and hasattr(subdef, "keys"):
need_pass_statement = False
c.m.sep()
subdef = definition["additionalProperties"]
with c.m.class_("Meta"):
if self.resolver.has_ref(subdef):
ref_name, _ = self.resolver.resolve_ref_definition(
d, subdef)
if ref_name is None:
raise CodegenError("$ref %r is not found",
subdef["$ref"])
self.write_field_one(c, d, ref_name, {},
"additional_field", subdef,
OrderedDict())
else:
self.write_field_one(c, d, "", subdef,
"additional_field", subdef,
OrderedDict())
if need_pass_statement:
c.m.stmt("pass")
def _callFUT(self, *args, **kwargs):
from dictknife import deepmerge
return deepmerge(*args, **kwargs)
d0 = {
"name": "foo",
"object": {
"x": 1,
"z": 1,
},
"children": [10],
"a": {
"b": {
"x": "y"
}
},
}
d1 = {
"name": "bar",
"object": {
"y": 2,
"z": 3,
},
"children": [1, 2, 3],
"a": {
"b": {
"c": "d",
"x": "z",
}
},
}
pp(deepmerge(d0, d1))
def expand(self, ctx: Context, data: dict, *, minimize=True) -> dict:
pool = self.pool
seen = self.seen
genid = self.genid
accessor = self.accessor
ref_walker = self.ref_walker
xxx_of_walker = self.xxx_of_walker
def _on_newdata(resolver, data):
for path, sd in xxx_of_walker.walk(data):
cases = sd[path[-1]]
for i, case in enumerate(cases):
if "$ref" in case:
continue
# todo: optimization
kid = genid()
assigned.add(kid)
pool[kid] = case
cases[i] = {"$ref": f"#/definitions/{kid}"}
seen[(resolver, f"definitions/{kid}")] = kid # xxx
return data
r = copy.deepcopy(data)
# todo: merge code with ..context:Context.get_uid()
# todo: drop description?
q = []
q.append((ctx.resolver, r))
assigned = set()
while q:
resolver, d = q.pop()
for path, sd in ref_walker.walk(d):
history = []
kid = genid()
k = (resolver, "/".join(map(str, path)).lstrip("#/"))
if k in seen:
assigned_kid = seen[k]
assigned.add(assigned_kid)
sd["$ref"] = f"#/definitions/{assigned_kid}"
continue
seen[k] = kid
history.append(k)
stack = [(resolver, sd["$ref"])]
found = False
while True:
resolver, ref = stack[-1]
k = (resolver, ref.lstrip("#/"))
if k in seen:
assigned_kid = seen[k]
assigned.add(assigned_kid)
sd["$ref"] = f"#/definitions/{assigned_kid}"
for k in history:
seen[k] = assigned_kid
break
seen[k] = kid
history.append(k)
sresolver, sref = resolver.resolve(ref)
sd = sresolver.access_by_json_pointer(sref)
if not hasattr(sd, "get") or "$ref" not in sd:
found = True
stack.append((sresolver, sd))
break
stack.append((sresolver, sd["$ref"]))
if not found:
continue
sresolver, sd = stack[-1]
new_sd = copy.deepcopy(sd)
if minimize:
for _, prop in list(self.description_walker.walk(new_sd)):
prop.pop("description")
pool[kid] = _on_newdata(sresolver, new_sd)
assigned.add(kid)
accessor.assign(r, path, f"#/definitions/{kid}")
q.append((sresolver, new_sd))
r = _on_newdata(ctx.resolver, r)
if not assigned:
return r
return deepmerge(
r,
{"definitions": {str(kid): pool[kid]
for kid in sorted(assigned)}})
