def merge(
*,
files: list,
dst: str,
style: str, # flavor?, strategy?
strict: bool = False,
wrap: str = None,
wrap_section: str = "definitions"
):
"""merge files"""
from dictknife.langhelpers import make_dict, as_jsonpointer
from dictknife import deepmerge
if style == "ref":
dstdir = dst and os.path.dirname(dst)
r = make_dict()
seen = {}
for src in files:
d = loading.loadfile(src)
for ns, sd in d.items():
for name in sd:
if ns not in r:
r[ns] = make_dict()
seen[ns] = make_dict()
if strict and name in r[ns]:
raise RuntimeError(
"{name} is already existed, (where={where} and {where2})".format(
name=name, where=seen[ns][name], where2=src
)
)
if dst is None:
where = ""
else:
where = os.path.relpath(src, start=dstdir)
r[ns][name] = {
"$ref": "{where}#/{ns}/{name}".format(
where=where, ns=ns, name=as_jsonpointer(name)
)
}
seen[ns][name] = src
elif style == "whole":
# TODO: strict support?
data = [loading.loadfile(src) for src in files]
r = deepmerge(*data, override=True)
else:
raise RuntimeError("invalid style: {}".format(style))
if wrap is not None:
wd = make_dict()
wd["type"] = "object"
wd["properties"] = make_dict()
for name in r.get(wrap_section) or {}:
wd["properties"][name] = {
"$ref": "#/{wrap_section}/{name}".format(
wrap_section=wrap_section, name=name
)
}
r[wrap_section][wrap] = wd
loading.dumpfile(r, dst)
def loadfile_with_jsonref(ref):
if "#/" not in ref:
return loading.loadfile(ref)
filename, query = ref.split("#")
doc = loading.loadfile(filename)
return access_by_json_pointer(doc, query)
def run(*, config: str) -> None:
c = loading.loadfile(config)
overwrite_file = os.environ.get("OVERWRITE_CONFIG")
if overwrite_file is not None:
c2 = loading.loadfile(overwrite_file)
c = deepmerge(c, c2, method="replace")
loading.dumpfile(c)
def run(*, schema: str, data: str, cache: str) -> None:
data = loading.loadfile(data)
try:
with open(cache) as rf:
validator = pickle.load(rf)
except FileNotFoundError:
schema = loading.loadfile(schema)
# jsonschema.Draft7Validator.check_schema(schema)
validator = jsonschema.Draft7Validator(schema)
with open(cache, "wb") as wf:
pickle.dump(validator, wf)
print(list(validator.iter_errors(data)))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--extracted", required=True)
parser.add_argument("--jsonschema", required=True)
parser.add_argument("-o",
"--output-format",
default="yaml",
choices=["json", "yaml"])
args = parser.parse_args()
r = merge(
extracted=loading.loadfile(args.extracted),
jsonschema=loading.loadfile(args.jsonschema),
)
loading.dumpfile(r, format=args.output_format)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("src", nargs="?", default=None)
args = parser.parse_args()
rows = loading.loadfile(args.src, format="tsv")
print("\n".join(to_table(rows)))
def run(src):
d = loading.loadfile(src)
for path, d in LooseDictWalkingIterator(["enum"]).iterate(d):
print(path[-2])
for x in d["enum"]:
print("- {}".format(x))
print("")
def run(*, config_path: str) -> None:
# Create SQS client
c = loading.loadfile(config_path)
sqs = boto3.client("sqs")
queue_url = c["QueueUrl"]
# Long poll for message on provided SQS queue
response = sqs.receive_message(
QueueUrl=queue_url,
AttributeNames=["SentTimestamp"],
MaxNumberOfMessages=10,
MessageAttributeNames=["All"],
WaitTimeSeconds=20,
VisibilityTimeout=1,
)
messages = response["Messages"]
print(len(messages))
will_be_deleted = []
for msg in messages:
print(f"{msg['MessageId']}: {msg['Body']}")
will_be_deleted.append({
"Id": msg["MessageId"],
"ReceiptHandle": msg["ReceiptHandle"]
})
# delete
# import inspect
# print(inspect.getfullargspec(sqs.delete_message_batch))
print(sqs.delete_message_batch(QueueUrl=queue_url,
Entries=will_be_deleted))
def run(filename: str) -> None:
schema = loading.loadfile(filename)
d = {
"config": {
"variables": {"env": "production"},
"providers": {"aws": {"region": "us-west-1"}},
}
}
d = {
"config": {
"variables": {"env": "production"},
"providers": {"aws": {"region": "us-west-2"}},
}
}
cls = jsonschema.Draft4Validator
cls.check_schema(schema)
for e in cls(schema).iter_errors(d):
print("cause?")
print(" context", e.context)
print(" cause", e.cause)
print("where?")
print(" instance_path", e.path)
print(" schema_path", e.schema_path)
print(" validator", e.validator)
def transform(
*,
src: str,
dst: str,
code: str,
functions: str,
input_format: str,
output_format: str,
format: str,
sort_keys: str
):
"""transform dict"""
from magicalimport import import_symbol
if code is not None:
transform = eval(code)
elif functions:
def transform(d):
for fn in functions:
if "." not in fn:
fn = "dictknife.transform:{}".format(fn)
d = import_symbol(fn)(d)
return d
else:
transform = lambda x: x # NOQA
input_format = input_format or format
data = loading.loadfile(src, input_format)
result = transform(data)
loading.dumpfile(
result, dst, format=output_format or input_format or format, sort_keys=sort_keys
)
def transform(*, src: str, dst: str, config: str, config_file: str, code: str,
functions: str, input_format: str, output_format: str,
format: str, sort_keys: str):
"""transform dict"""
from magicalimport import import_symbol
from dictknife import deepmerge
if code is not None:
transform = eval(code)
elif functions:
def transform(d):
for fn in functions:
if "." not in fn:
fn = "dictknife.transform:{}".format(fn)
d = import_symbol(fn)(d)
return d
else:
transform = lambda x: x # NOQA
input_format = input_format or format
kwargs = loading.loads(config, format=input_format)
if config_file:
with open(config_file) as rf:
kwargs = deepmerge(kwargs, loading.load(rf, format=input_format))
data = loading.loadfile(src, input_format)
result = transform(data, **kwargs)
loading.dumpfile(result,
dst,
format=output_format or format,
sort_keys=sort_keys)
def examples(
*,
src: str,
dst: str = None,
ref: str,
limit: int,
input_format: str,
output_format: str,
format: str,
use_expand: bool = False,
):
"""output sample value from swagger's spec"""
from dictknife.jsonknife import extract_example
from dictknife.jsonknife.accessor import access_by_json_pointer
if use_expand:
from dictknife.jsonknife import bundle, expand
if ref is not None:
src = "{prefix}#/{name}".format(prefix=src, name=ref.lstrip("#/"))
data = bundle(src, format=input_format or format)
data = expand(None, doc=data)
else:
data = loading.loadfile(src, format=input_format or format)
if src and "#/" in src:
_, ref = src.split("#/", 1)
if ref is not None:
data = access_by_json_pointer(data, ref)
d = extract_example(data, limit=limit)
loading.dumpfile(d, dst, format=output_format or format or "json")
def run(*, config_path: str):
"""Exercise send_sqs_message()"""
# Assign this value before running the program
c = loading.loadfile(config_path)
sqs_queue_url = c["QueueUrl"]
sqs_client = boto3.client("sqs")
# Send some SQS messages
entries = []
for i in range(1, 6):
entries.append({
"Id": f"m{i}",
"MessageBody": f"SQS message #{i}",
"DelaySeconds": 10
})
try:
assert (
entries
), "!! An error occurred (AWS.SimpleQueueService.EmptyBatchRequest) when calling the SendMessageBatch operation: There should be at least one SendMessageBatchRequestEntry in the request."
response = sqs_client.send_message_batch(QueueUrl=sqs_queue_url,
Entries=entries)
print(response)
except ClientError as e:
print("!!", e)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("src", nargs="?", default=None)
parser.add_argument("--package", default=None)
parser.add_argument("--position", default=None)
parser.add_argument("--writer", default="goaway.writer:Writer")
parser.add_argument("--emitter", default="goaway.emitter:Emitter")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
loading.setup()
r = get_repository(
writer_cls=import_symbol(args.writer),
emitter_cls=import_symbol(args.emitter),
)
data = loading.loadfile(args.src)
package = r.package(args.package or "main")
walk(data, package, r)
d = r.resolve_package_path(args.position, package)
r.emitter.emit_package(package, d=d)
def run(filename: str) -> str:
d = loading.loadfile(filename)
w = DictWalker(["vars", "exec"])
for path, w in w.walk(d):
# path: services/<service_name>/vars/exec
service_name = path[-3]
exec_path = w["exec"]
if exec_path == "./enduser_server":
continue
if exec_path == "./tuner_server":
continue
if _normalize(_normalize2(exec_path)).startswith(
_normalize(_normalize2(service_name))
):
continue
loading.dumpfile(
{
"must_include_part": _normalize(_normalize2(service_name)),
"ref": f"#/{'/'.join(path[:-2])}",
"exec_path": exec_path,
},
format="json",
)
print("")
def run(src):
d = loading.loadfile(src)
for path, d in LooseDictWalkingIterator(["enum"]).iterate(d):
print(path[-2])
for x in d["enum"]:
print("- {}".format(x))
print("")
def run(filename: str) -> None:
d = loading.loadfile(filename)
ctx = Context()
visit(ctx, d)
print("@", ctx)
print("----------------------------------------")
print(emit(ctx))
def run(*, path: str, disable_docstring) -> None:
d = loading.loadfile(path)
m = Module()
a = Accessor(d, m, disable_docstring=disable_docstring)
m.import_("typing", as_="t")
m.sep()
m.stmt("AnyService = t.Any # TODO")
m.stmt("AnyResource = t.Any # TODO")
m.sep()
for rname, resource in a.resources.items():
with m.class_(titleize(rname), ""):
with m.method("__init__", "resource: AnyResource"):
m.stmt("self.internal = resource")
m.stmt("# methods")
for mname, method, params in a.iterate_methods(resource):
with m.method(mname, params):
a.emit_docstring(method["description"])
m.stmt(
f"""# {method["httpMethod"]}: {method["flatPath"]}""")
m.stmt(f"""# id: {method["id"]}""")
m.stmt(f"return self.internal.{mname}({params})")
m.stmt("# nested resources")
for srname, subresource in a.iterate_nested_resources(resource):
with m.method(srname):
m.stmt(f"return self.internal.{srname}({params})")
# m.stmt("# nested resources")
# for mname, subresource in resource.get("resources", {}).items():
# params = LParams()
# for is_positional, (pname, parameter) in itertools.zip_longest(subresource.get("parameterOrder", []), subresource.get("parameters", {}).items()):
# if is_positional:
# params.append(pname) # TODO type:
# else:
# params[pname] = None # TODO type:
# with m.method(mname, params):
# docstring(subresource["description"])
# m.stmt(f"""# id: {subresource["id"]}""")
# m.stmt(f"return self.{mname}({params})")
with m.class_("Service"):
with m.method("__init__", "service: AnyService"):
m.stmt("self.internal = service")
for rname in a.resources.keys():
with m.method(rname, return_type=titleize(rname)):
m.stmt(f"return {titleize(rname)}(self.internal.{rname}())")
with m.def_("build", "*args", "**kwargs", return_type="Service"):
m.stmt("# TODO: use the signature of googleapiclient.discovery.build")
m.submodule().from_("googleapiclient.discovery", "build")
m.stmt(
f"return Service(build({a.name!r}, {a.version!r}, *args, **kwargs))"
)
print(m)
def flatten(*, src: str, dst: str, input_format: str, output_format: str, format: str):
"""flatten jsonschema sub definitions"""
from dictknife.swaggerknife.flatten import flatten
input_format = input_format or format
data = loading.loadfile(src, format=input_format)
d = flatten(data)
loading.dumpfile(d, dst, format=output_format or format)
def json2swagger(
*,
files,
dst: str,
output_format: str,
name: str,
detector,
emitter,
annotate,
emit,
with_minimap: bool,
without_example: bool
):
from prestring import Module
from dictknife import DictWalker
if annotate is not None:
annotate = loading.loadfile(annotate)
else:
annotate = {}
ns = "dictknife.swaggerknife.json2swagger"
detector = import_symbol(detector, ns=ns)()
emitter = import_symbol(emitter, ns=ns)(annotate)
info = None
for src in files:
data = loading.loadfile(src)
info = detector.detect(data, name, info=info)
if emit == "info":
loading.dumpfile(info, filename=dst)
else:
m = Module(indent=" ")
m.stmt(name)
emitter.emit(info, m)
if with_minimap:
print("# minimap ###")
print("# *", end="")
print("\n# ".join(str(m).split("\n")))
if without_example:
for _, d in DictWalker(["example"]).walk(emitter.doc):
d.pop("example")
loading.dumpfile(emitter.doc, filename=dst, format=output_format)
def data(self):
data = deepmerge(
*[loading.loadfile(d) for d in self.data_path_list], override=True
)
if self.format is not None:
data = deepmerge(
data, loading.load(sys.stdin, format=self.format), override=True
)
return data
def run(src):
d = loading.loadfile(src)
for path, d in LooseDictWalkingIterator(["type"]).iterate(d):
if d["type"] == "boolean":
if "name" in d:
name = d["name"]
else:
name = path[-2]
print(name, d.get("description"))
def run(filename: str, output: t.Optional[str] = None):
d = loading.loadfile(filename)
for path, sd in DictWalker(["allOf"]).walk(d):
parent = d
for name in path[:-2]:
parent = parent[name]
assert parent[path[-2]] == sd
parent[path[-2]] = sd.pop("allOf")[0]
loading.dumpfile(d, output)
def cut(*, src, dst, refs):
from dictknife.accessing import Accessor
d = loading.loadfile(src)
accessor = Accessor(make_dict)
for ref in refs:
if ref.startswith("#/"):
ref = ref[2:]
accessor.maybe_remove(d, ref.split("/"))
loading.dumpfile(d, dst)
def annotate(filename: str, *, use_fullname: bool = False) -> None:
from dictknife import loading
from detector import detect, generate_annotations
d = loading.loadfile(filename)
result = detect(d)
r = generate_annotations(result,
use_fullname=use_fullname,
toplevel_name="toplevel")
loading.dumpfile(r)
def run(*, config: str) -> None:
import sys
from dictknife import loading
d = loading.loadfile(config)
try:
c = Config().load(d)
except marshmallow.ValidationError as e:
print(e.normalized_messages(), file=sys.stderr)
sys.exit(1)
print(c)
def run(*, config: str) -> None:
import sys
from dictknife import loading
d = loading.loadfile(config)
try:
c = Config(**d)
except pydantic.ValidationError as e:
print(e.json(), file=sys.stderr)
sys.exit(1)
print(c)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("src", nargs="?", default=None)
parser.add_argument("--package", default="autogenerated")
args = parser.parse_args()
loading.setup()
data = loading.loadfile(args.src)
m = go.Module()
m.package(args.package)
emit(data, m)
print(m)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("src", nargs="?", default=None)
parser.add_argument("--package", default="autogenerated")
args = parser.parse_args()
loading.setup()
data = loading.loadfile(args.src)
m = go.Module()
m.package(args.package)
emit(data, m)
print(m)
def _load_data(self, name_or_data, *, cache):
if name_or_data is None:
return {}
elif isinstance(name_or_data, (list, tuple)):
return deepmerge(
*[self._load_data(d, cache=cache) for d in name_or_data])
elif hasattr(name_or_data, "get"):
return name_or_data
else:
r = cache.get(name_or_data)
if r is None:
r = cache[name_or_data] = loading.loadfile(name_or_data)
return r
def run(filename: str, *, verbose: bool = False, profile: bool = False) -> None:
if profile:
import cProfile
import pstats
prof = cProfile.Profile()
prof.enable()
d = loading.loadfile(filename)
main(d, verbose=verbose)
if profile:
prof.disable()
s = pstats.Stats(prof)
s.dump_stats("metashape-inputs-openapi.prof")
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--data", action="append", help="support yaml, json, toml", default=[])
parser.add_argument("template")
parser.add_argument("--input-format", default=None)
parser.add_argument("--output-format", default="raw")
parser.add_argument("--dst", default=None)
args = parser.parse_args()
loading.setup()
data = deepmerge(*[loading.loadfile(d) for d in args.data], override=True)
if args.input_format is not None:
data = deepmerge(data, loading.load(sys.stdin, format=args.input_format), override=True)
result = run(args.template, data)
loading.dumpfile(result, args.dst, format=args.output_format)
def run(
filename: str,
*,
schema: t.Union[None, str, dict] = None,
guess_schema: bool,
always_success: bool,
output: OutputType,
) -> int:
filepath = os.path.abspath(filename)
s = streams.from_filename(filepath)
if guess_schema:
wlogger = LoggerWithCollectMessage(logger, {})
schema = guess.guess_schema(
filepath,
code=".schemalint.py",
current=os.path.dirname(filename),
logger=wlogger,
)
s = streams.append_messages(s, messages=wlogger.messages)
if schema is not None:
if isinstance(schema, str):
if schema.startswith(("https://", "http://")):
import requests
schema = requests.get(schema).json()
else:
from dictknife import loading
schema = loading.loadfile(schema)
validator = get_validator(schema,
check_schema=True) # TODO: skip_check option
s = streams.with_validator(s, validator)
formatter = get_formatter(filepath,
lookup=s.context.lookup,
output_type=output)
success = True
for ev in s:
if not always_success and not ev.has_soft_error:
success = False
print(formatter.format(ev))
return 0 if success else 1
def run(
*,
file: str,
pattern: str = None,
ignore: str = None,
format: str = "json",
shy: bool = False,
with_request_type: bool = False,
with_response_type: bool = False,
ignore_cookies: bool = False,
include_all: bool = False,
) -> None:
import re
pattern_rx = pattern and re.compile(pattern)
ignore_rx = ignore and re.compile(ignore)
d = loading.loadfile(file, format=format)
for domain, entries in classify(d).items():
if pattern_rx and pattern_rx.search(domain) is None:
continue
if ignore_rx and ignore_rx.search(domain) is not None:
continue
print("##", domain)
print("")
print("```yaml")
r = aggregate(entries, shy=shy)
loading.dumpfile(
transform(
r,
with_request_type=with_request_type,
with_response_type=with_response_type,
with_cookies=not ignore_cookies,
include_all=include_all,
)
)
print("```")
print("")
def run(src):
loading.setup()
d = loading.loadfile(src)
store = collect(d["package"])
r = arrange(store)
loading.dumpfile(r, format="toml")
def run(*, src: str) -> None:
data = loading.loadfile(src)
loading.dumpfile(data, format="yaml")
def run(path: str) -> None:
rows = loading.loadfile(path, format="csv")
import itertools
rows = itertools.islice(rows, 3)
loading.dumpfile(rows, format="json")
def run(args):
config = loading.loadfile(args.config)
template = config["template"].strip()
c = Context(args, config.get("defaults") or {})
print(template.format(c=c))
from nbreversible import (code, )
import pandas as pd
from dictknife import loading
# %matplotlib inline
with code():
df = pd.DataFrame.from_dict(loading.loadfile("points.json")).set_index("name")
df
with code():
ax = df.plot(kind="scatter", x="x", y="y", s=40)
for _, row in df.iterrows():
ax.annotate(
row.name, (row.x, row.y),
color="k",
weight="semibold",
size="medium",
horizontalalignment="left",
verticalalignment="top"
)
from dictknife import loading
from zenmai.naming import snakecase
loading.setup()
path = "./app.yaml"
d = loading.loadfile(path)
print("\n".join([snakecase(v["name"]) for v in d["X"].values()]))
from dictknife import loading
import jsonschema
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--schema", required=True)
parser.add_argument("--data", required=True)
args = parser.parse_args()
schema = loading.loadfile(args.schema)
jsonschema.Draft4Validator.check_schema(schema)
data = loading.loadfile(args.data)
for err in jsonschema.Draft4Validator(schema).iter_errors(data):
print(err)
from dictknife.transform import flatten
from dictknife import loading
d = loading.loadfile("00person.json")
loading.dumpfile(d, format="json")
loading.dumpfile(flatten(d), format="json")
print("----------------------------------------")
d = loading.loadfile("02person.json")
loading.dumpfile(d, format="json")
loading.dumpfile(flatten(d), format="json")
def main(*, src: str, dst: str) -> None:
src = loading.loadfile(src)
dst = loading.loadfile(dst)
patches = list(unpatch.unpatch(src, dst, verbose=True))
loading.dumpfile(patches, format="json")
import sys
import urllib.parse as p
from collections import defaultdict
from dictknife import loading
if len(sys.argv) == 1:
# sys.argv.append("data.har")
sys.argv.append(None)
d = loading.loadfile(sys.argv[1], format="json")
r = defaultdict(list)
for entry in d["log"]["entries"]:
parsed = p.urlparse(entry["request"]["url"])
if "application/json" in entry["response"]["content"]["mimeType"].lower():
r[parsed.netloc].append(entry)
loading.dumpfile(r)
def run(*, file: str, name: str = "response") -> None:
d = loading.loadfile(file)
loading.dumpfile(makeschema(d, name=name))
def main(src):
d = loading.loadfile(src)
m = GraphQLModule()
emit(m, d)
print(m)
def run(src):
d = loading.loadfile(src)
target = d["module"]["model"]
dig(target)
from dictknife import loading, pp
from schema import Conf
loading.setup()
data = loading.loadfile("./config-sample.json")
data, err = Conf().load(data)
if err:
pp(err)
else:
loading.dumpfile(data)
