def generateGraph(logger):
'''generate a directed graph from the modules config
generate a networkX.Graph object by reading the contents
of the ``config/modules/`` folder.
Parameters
----------
logger : {logging.logger}
logging element
Returns
-------
networkX.Graph object
Graph of which object is created when
'''
try:
graph = nx.DiGraph()
folder = '../config/modules'
files = [f for f in os.listdir(folder) if f.endswith('.json')]
for f in files:
data = jsonref.load(open(os.path.join(folder, f)))
inp = list(data['inputs'].keys())
out = list(data['outputs'].keys())
f = f.replace('.json', '')
graph.add_node(f, type='module', summary='')
# Add the incoming edges
for n in inp:
if n not in graph.nodes:
summary = jsonref.dumps(data['inputs'][n])
graph.add_node(n,
type=data['inputs'][n]['type'],
summary=summary)
graph.add_edge(n, f)
# Add the outgoing edges
for n in out:
if n not in graph.nodes:
summary = jsonref.dumps(data['outputs'][n])
graph.add_node(n,
type=data['outputs'][n]['type'],
summary=summary)
graph.add_edge(f, n)
except Exception as e:
logger.error('Unable to generate the graph: {}'.format(e))
return graph
def get_full_schema(schema_dir):
print(schema_dir)
os.chdir(schema_dir)
fn = "map.json"
uri = "file:///{}/".format(schema_dir)
with open(fn) as f:
j = jsonref.load(f, base_uri=uri)
jsn = jsonref.dumps(j, indent=4, sort_keys=False)
full_schema = jsonref.dumps(j, indent=4, sort_keys=False)
with open(r"C:\Temp\mapfile.json", "w") as f:
f.write(full_schema)
return full_schema
def test_dumps(self):
json = """[1, 2, {"$ref": "#/0"}, 3]"""
loaded = loads(json)
# The string version should load the reference
assert str(loaded) == "[1, 2, 1, 3]"
# Our dump function should write the original reference
assert dumps(loaded) == json
def test_dumps(self):
json = """[1, 2, {"$ref": "#/0"}, 3]"""
loaded = loads(json)
# The string version should load the reference
assert str(loaded) == "[1, 2, 1, 3]"
# Our dump function should write the original reference
assert dumps(loaded) == json
def get_schema(monkeypatch, schema_in_bytes):
"""Get json schema and replace $refs.
For the resolving of the $ref we have to catch the request.get and
get the referenced json schema directly from the resource.
:param monkeypatch: https://docs.pytest.org/en/stable/monkeypatch.html
:schema_in_bytes: schema in bytes.
:returns: resolved json schema.
"""
# apply the monkeypatch for requests.get to mocked_requests_get
monkeypatch.setattr(requests, "get", mocked_requests_get)
schema = jsonref.loads(schema_in_bytes.decode('utf8'))
# Replace all remaining $refs
while schema != jsonref.loads(jsonref.dumps(schema)):
schema = jsonref.loads(jsonref.dumps(schema))
return schema
def render(self, *args, **kwargs):
output = super().render(*args, **kwargs)
data = jsonref.loads(output)
# add api level metadata
info = data.get("info", {})
info.update({
"x-logo": {
"url": "https://i.imgur.com/tVsRNxJ.pngg",
"altText": "Penn Labs Logo",
"href": "https://pennlabs.org/",
},
"contact": {
"name": f"{settings.BRANDING_SITE_NAME} Support",
"email": re.search(r"<(.*)>", settings.FROM_EMAIL).group(1),
},
"description": API_DESCRIPTION.strip(),
})
data["info"] = info
# categorize api endpoints
categories = set()
for path in data["paths"]:
category = re.match(r"/api/(.*?)/",
path).group(1).replace("_", " ").title()
categories.add(category)
for key in data["paths"][path]:
oper_id = data["paths"][path][key]["operationId"]
data["paths"][path][key]["operationId"] = re.sub(
r"(?<=[a-z])([A-Z])", r" \1", oper_id).title()
data["paths"][path][key]["tags"] = [category]
# order tag groups
categories = list(sorted(categories))
data["tags"] = [{"name": cat, "description": ""} for cat in categories]
data["x-tagGroups"] = [{
"name": settings.BRANDING_SITE_NAME,
"tags": categories
}]
return jsonref.dumps(
data, indent=4 if settings.DEBUG else None).encode("utf-8")
def dump_schema(schema):
return jsonref.dumps(schema)
import sys
import copy
try:
r = requests.get(sys.argv[1])
release = r.json()
print("Fetched schema from URL")
except:
print("Using local release schema")
with open('release-schema.json', 'r') as f:
release = json.loads(f.read(), object_pairs_hook=collections.OrderedDict)
release = JsonRef.replace_refs(release)
print(jsonref.dumps(release, indent=3))
# Based on https://stackoverflow.com/questions/30734682/extracting-url-and-anchor-text-from-markdown-using-python
INLINE_LINK_RE = re.compile(r'\[([^\]]+)\]\(([^)]+)\)')
def find_md_links(md):
links = dict(INLINE_LINK_RE.findall(md))
return links
def remove_links(text, links):
for key, link in links.items():
text = text.replace("[" + key + "](" + link + ")", key)
return text
import json
import jsonref
#----------------------------
# cityjson.schema.json
#----------------------------
root_schema = os.path.abspath('../schemas/cityjson.schema.json')
fins = open(root_schema)
abs_path = os.path.abspath(os.path.dirname(root_schema))
base_uri = 'file://{}/'.format(abs_path)
js = jsonref.loads(fins.read(), jsonschema=True, base_uri=base_uri)
# -- output stitched_schema
json_str = jsonref.dumps(js, indent=2)
# json_str = jsonref.dumps(js, separators=(',',':')) #-- this doesn't work: WTF
opath = os.path.abspath('temp.json')
f = open(opath, "w")
f.write(json_str)
f2 = open('./temp.json')
j2 = json.loads(f2.read())
json_str2 = json.dumps(j2, separators=(',', ':'))
f = open("../schemas/cityjson.min.schema.json", "w")
f.write(json_str2)
#----------------------------
# cityjsonfeature.schema.json
#----------------------------
root_schema = os.path.abspath('../schemas/cityjsonfeature.schema.json')
jsonschema2rst %input_folder% %output_folder%
"""
from pprint import pprint
import jsonref
from jsonref import JsonRef
# Sample JSON data, like from json.load
document = {"data": ["a", "b", "c"], "reference": {"$ref": "#/data/1"}}
import os
d = r"D:\GitHub\mappyfile\mappyfile\schemas"
os.chdir(d)
fn = "map.json"
uri = "file:///D:/GitHub/mappyfile/mappyfile/schemas/"
with open(fn) as f:
j = jsonref.load(f, base_uri=uri)
jsn = jsonref.dumps(j, indent=4, sort_keys=False)
with open(r"D:\Temp\mapfile.json", "w") as f:
f.write(jsonref.dumps(j, indent=4, sort_keys=False))
## The JsonRef.replace_refs class method will return a copy of the document
## with refs replaced by :class:`JsonRef` objects
#pprint(JsonRef.replace_refs(document))
print("Done!")
Docs
C:\VirtualEnvs\mappyfile\Scripts\activate.bat
SET "input_folder=D:\GitHub\mappyfile\mappyfile\schemas"
SET "output_folder=D:\GitHub\mappyfile\docs\schemas\"
jsonschema2rst %input_folder% %output_folder%
"""
from pprint import pprint
import jsonref
from jsonref import JsonRef
# Sample JSON data, like from json.load
document = {"data": ["a", "b", "c"], "reference": {"$ref": "#/data/1"}}
import os
d = r"D:\GitHub\mappyfile\mappyfile\schemas"
os.chdir(d)
fn = "map.json"
uri = "file:///D:/GitHub/mappyfile/mappyfile/schemas/"
with open(fn) as f:
j = jsonref.load(f, base_uri=uri)
print(jsonref.dumps(j, indent=4, sort_keys=False))
## The JsonRef.replace_refs class method will return a copy of the document
## with refs replaced by :class:`JsonRef` objects
#pprint(JsonRef.replace_refs(document))
def dump_schema(schema):
return jsonref.dumps(schema)
def test_expecations(self):
"""Just one big test"""
# Save some data that contains a $ref
data = JsonExpandOMatic(path="funk").expand(
data=json.loads(TestJsonRefKeeper._raw_data1))
assert data == {"root": {"$ref": "funk/root.json"}}
# Load the previously expanded data
data = JsonExpandOMatic(path="funk").contract(root_element="root")
assert data == {
"foo": {
"bar": {
"baz": 1234
}
},
"stuff": {
"$ref": "#/foo/bar"
}
}
data["foo"]["bar"]["baz"] = -1 # Make a change
assert data == {
"foo": {
"bar": {
"baz": -1
}
},
"stuff": {
"$ref": "#/foo/bar"
}
}
# Use jsonrefkeeper to resolve the $ref entry in data['stuff']
data = jsonrefkeeper.parse(data)
# Both jsonref and jsonrefkeeper resolve the $ref such that
# data['stuff'] and data['foo']['bar'] point to the same object: {'baz': -1}
# Changing either data['stuff']['baz'] or data['foo']['bar']['baz']
# will have the same result.
assert data == {"foo": {"bar": {"baz": -1}}, "stuff": {"baz": -1}}
data["foo"]["bar"]["baz"] = -2
assert data == {"foo": {"bar": {"baz": -2}}, "stuff": {"baz": -2}}
# json.dumps(data, indent=None) # TypeError: Object of type 'dict' is not JSON serializable
assert (json.dumps(data, indent=2) == ""
'{\n "foo": {\n "bar": {\n "baz": -2\n }\n },\n'
' "stuff": {\n "baz": -2\n }\n}')
# Using jsonref to dump the data with no indent will preserve the original $ref.
assert jsonref.dumps(
data, indent=None
) == '{"foo": {"bar": {"baz": -2}}, "stuff": {"$ref": "#/foo/bar"}}'
# However providing an indent will cause the underlying proxy object to be resolved and we lose the $ref.
assert (jsonref.dumps(data, indent=2) == ""
'{\n "foo": {\n "bar": {\n "baz": -2\n }\n },\n'
' "stuff": {\n "baz": -2\n }\n}')
# This is equivalent to jsonref.dumps(data, indent=None) & probably a little slower for large data structures.
assert (
jsonrefkeeper.dumps(data, indent=None) == ""
'{"foo": {"bar": {"baz": -2}}, "stuff": {"$ref": "#/foo/bar"}}')
# This is what jsonrefkeeper was created for. We don't have to choose between
# keeping the refs and having nicely indented output.
assert (jsonrefkeeper.dumps(data, indent=2) == ""
'{\n "foo": {\n "bar": {\n "baz": -2\n }\n },\n'
' "stuff": {\n "$ref": "#/foo/bar"\n }\n}')
"""
from pprint import pprint
import jsonref
from jsonref import JsonRef
# Sample JSON data, like from json.load
document = {
"data": ["a", "b", "c"],
"reference": {"$ref": "#/data/1"}
}
import os
d = r"D:\GitHub\mappyfile\mappyfile\schemas"
os.chdir(d)
fn = "map.json"
uri = "file:///D:/GitHub/mappyfile/mappyfile/schemas/"
with open(fn) as f:
j = jsonref.load(f, base_uri=uri)
jsn = jsonref.dumps(j, indent=4, sort_keys=False)
with open(r"D:\Temp\mapfile.json", "w") as f:
f.write(jsonref.dumps(j, indent=4, sort_keys=False))
## The JsonRef.replace_refs class method will return a copy of the document
## with refs replaced by :class:`JsonRef` objects
#pprint(JsonRef.replace_refs(document))
print("Done!")
