def get(url):
"""
USE json.net CONVENTIONS TO LINK TO INLINE OTHER JSON
"""
if not _Log:
_late_import()
if url.find("://") == -1:
_Log.error("{{url}} must have a prototcol (eg http://) declared",
url=url)
base = URL("")
if url.startswith("file://") and url[7] != "/":
if os.sep == "\\":
base = URL("file:///" +
os.getcwd().replace(os.sep, "/").rstrip("/") + "/.")
else:
base = URL("file://" + os.getcwd().rstrip("/") + "/.")
elif url[url.find("://") + 3] != "/":
_Log.error("{{url}} must be absolute", url=url)
phase1 = _replace_ref(wrap({"$ref": url}),
base) # BLANK URL ONLY WORKS IF url IS ABSOLUTE
try:
phase2 = _replace_locals(phase1, [phase1])
return wrap(phase2)
except Exception, e:
_Log.error("problem replacing locals in\n{{phase1}}",
phase1=phase1,
cause=e)
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise _Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = dot.get_attr(new_value, ref.fragment)
if DEBUG:
_Log.note("Replace {{ref}} with {{new_value}}",
ref=ref,
new_value=new_value)
if not output:
output = new_value
else:
output = unwrap(set_default(output, new_value))
if DEBUG:
_Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise _Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = dot.get_attr(new_value, ref.fragment)
if DEBUG:
_Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value)
if not output:
output = new_value
else:
output = unwrap(set_default(output, new_value))
if DEBUG:
_Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref, raw_ref, node["$ref"] = URL(node["$ref"]), node["$ref"], None
# RECURS
return_value = node
candidate = {}
for k, v in node.items():
new_v = _replace_ref(v, url)
candidate[k] = new_v
if new_v is not v:
return_value = candidate
if not ref:
return return_value
else:
node = return_value
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
node["$ref"] = ref
return node
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = new_value[ref.fragment]
if isinstance(new_value, Mapping):
return set_default({}, node, new_value)
elif node.keys() and new_value == None:
return node
else:
return wrap(new_value)
elif isinstance(node, list):
candidate = [_replace_ref(n, url) for n in node]
if all(p[0] is p[1] for p in zip(candidate, node)):
return node
return candidate
return node
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref, raw_ref, node["$ref"] = URL(node["$ref"]), node["$ref"], None
# RECURS
return_value = node
candidate = {}
for k, v in node.items():
new_v = _replace_ref(v, url)
candidate[k] = new_v
if new_v is not v:
return_value = candidate
if not ref:
return return_value
else:
node = return_value
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
node["$ref"] = ref
return node
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise Log.error("unknown protocol {{scheme}}", scheme= ref.scheme)
if ref.fragment:
new_value = new_value[ref.fragment]
if isinstance(new_value, Mapping):
return set_default({}, node, new_value)
elif node.keys() and new_value == None:
return node
else:
return wrap(new_value)
elif isinstance(node, list):
candidate = [_replace_ref(n, url) for n in node]
if all(p[0] is p[1] for p in zip(candidate, node)):
return node
return candidate
return node
def expand(doc, doc_url):
"""
ASSUMING YOU ALREADY PULED THE doc FROM doc_url, YOU CAN STILL USE THE
EXPANDING FEATURE
"""
if doc_url.find("://") == -1:
Log.error("{{url}} must have a prototcol (eg http://) declared",
url=doc_url)
phase1 = _replace_ref(
doc, URL(doc_url)) # BLANK URL ONLY WORKS IF url IS ABSOLUTE
phase2 = _replace_locals(phase1, [phase1])
return wrap(phase2)
def test_save_then_load(self):
test = {
"data": [
{"a": "b"}
],
"query": {
"meta": {"save": True},
"from": TEST_TABLE,
"select": "a"
},
"expecting_list": {
"meta": {
"format": "list"
},
"data": ["b"]
}
}
settings = self.utils.fill_container(test)
bytes = convert.unicode2utf8(convert.value2json({
"from": settings.index,
"select": "a",
"format": "list"
}))
expected_hash = convert.bytes2base64(hashlib.sha1(bytes).digest()[0:6]).replace("/", "_")
wrap(test).expecting_list.meta.saved_as = expected_hash
self.utils.send_queries(test)
#ENSURE THE QUERY HAS BEEN INDEXED
container = elasticsearch.Index(index="saved_queries", settings=settings)
container.flush()
Thread.sleep(seconds=5)
url = URL(self.utils.service_url)
response = self.utils.try_till_response(url.scheme+"://"+url.host+":"+unicode(url.port)+"/find/"+expected_hash, data=b'')
self.assertEqual(response.status_code, 200)
self.assertEqual(response.all_content, bytes)
def get(url):
if not Log:
_late_import()
"""
USE json.net CONVENTIONS TO LINK TO INLINE OTHER JSON
"""
if url.find("://") == -1:
Log.error("{{url}} must have a prototcol (eg http://) declared",
url=url)
if url.startswith("file://") and url[7] != "/":
# RELATIVE
if os.sep == "\\":
url = "file:///" + os.getcwd().replace(os.sep, "/") + "/" + url[7:]
else:
url = "file://" + os.getcwd() + "/" + url[7:]
if url[url.find("://") + 3] != "/":
Log.error("{{url}} must be absolute", url=url)
doc = wrap({"$ref": url})
phase1 = _replace_ref(doc,
URL("")) # BLANK URL ONLY WORKS IF url IS ABSOLUTE
phase2 = _replace_locals(phase1, [phase1])
return wrap(phase2)