def get(url): """ USE json.net CONVENTIONS TO LINK TO INLINE OTHER JSON """ if not _Log: _late_import() if url.find("://") == -1: _Log.error("{{url}} must have a prototcol (eg http://) declared", url=url) base = URL("") if url.startswith("file://") and url[7] != "/": if os.sep == "\\": base = URL("file:///" + os.getcwd().replace(os.sep, "/").rstrip("/") + "/.") else: base = URL("file://" + os.getcwd().rstrip("/") + "/.") elif url[url.find("://") + 3] != "/": _Log.error("{{url}} must be absolute", url=url) phase1 = _replace_ref(wrap({"$ref": url}), base) # BLANK URL ONLY WORKS IF url IS ABSOLUTE try: phase2 = _replace_locals(phase1, [phase1]) return wrap(phase2) except Exception, e: _Log.error("problem replacing locals in\n{{phase1}}", phase1=phase1, cause=e)
def _replace_ref(node, url): if url.path.endswith("/"): url.path = url.path[:-1] if isinstance(node, Mapping): ref = None output = {} for k, v in node.items(): if k == "$ref": ref = URL(v) else: output[k] = _replace_ref(v, url) if not ref: return output node = output if not ref.scheme and not ref.path: # DO NOT TOUCH LOCAL REF YET output["$ref"] = ref return output if not ref.scheme: # SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH # REQUIRES THE CURRENT DOCUMENT'S SCHEME ref.scheme = url.scheme # FIND THE SCHEME AND LOAD IT if ref.scheme in scheme_loaders: new_value = scheme_loaders[ref.scheme](ref, url) else: raise _Log.error("unknown protocol {{scheme}}", scheme=ref.scheme) if ref.fragment: new_value = dot.get_attr(new_value, ref.fragment) if DEBUG: _Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value) if not output: output = new_value else: output = unwrap(set_default(output, new_value)) if DEBUG: _Log.note("Return {{output}}", output=output) return output elif isinstance(node, list): output = [_replace_ref(n, url) for n in node] # if all(p[0] is p[1] for p in zip(output, node)): # return node return output return node
def _replace_ref(node, url): if url.path.endswith("/"): url.path = url.path[:-1] if isinstance(node, Mapping): ref, raw_ref, node["$ref"] = URL(node["$ref"]), node["$ref"], None # RECURS return_value = node candidate = {} for k, v in node.items(): new_v = _replace_ref(v, url) candidate[k] = new_v if new_v is not v: return_value = candidate if not ref: return return_value else: node = return_value if not ref.scheme and not ref.path: # DO NOT TOUCH LOCAL REF YET node["$ref"] = ref return node if not ref.scheme: # SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH # REQUIRES THE CURRENT DOCUMENT'S SCHEME ref.scheme = url.scheme # FIND THE SCHEME AND LOAD IT if ref.scheme in scheme_loaders: new_value = scheme_loaders[ref.scheme](ref, url) else: raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme) if ref.fragment: new_value = new_value[ref.fragment] if isinstance(new_value, Mapping): return set_default({}, node, new_value) elif node.keys() and new_value == None: return node else: return wrap(new_value) elif isinstance(node, list): candidate = [_replace_ref(n, url) for n in node] if all(p[0] is p[1] for p in zip(candidate, node)): return node return candidate return node
def expand(doc, doc_url): """ ASSUMING YOU ALREADY PULED THE doc FROM doc_url, YOU CAN STILL USE THE EXPANDING FEATURE """ if doc_url.find("://") == -1: Log.error("{{url}} must have a prototcol (eg http://) declared", url=doc_url) phase1 = _replace_ref( doc, URL(doc_url)) # BLANK URL ONLY WORKS IF url IS ABSOLUTE phase2 = _replace_locals(phase1, [phase1]) return wrap(phase2)
def test_save_then_load(self): test = { "data": [ {"a": "b"} ], "query": { "meta": {"save": True}, "from": TEST_TABLE, "select": "a" }, "expecting_list": { "meta": { "format": "list" }, "data": ["b"] } } settings = self.utils.fill_container(test) bytes = convert.unicode2utf8(convert.value2json({ "from": settings.index, "select": "a", "format": "list" })) expected_hash = convert.bytes2base64(hashlib.sha1(bytes).digest()[0:6]).replace("/", "_") wrap(test).expecting_list.meta.saved_as = expected_hash self.utils.send_queries(test) #ENSURE THE QUERY HAS BEEN INDEXED container = elasticsearch.Index(index="saved_queries", settings=settings) container.flush() Thread.sleep(seconds=5) url = URL(self.utils.service_url) response = self.utils.try_till_response(url.scheme+"://"+url.host+":"+unicode(url.port)+"/find/"+expected_hash, data=b'') self.assertEqual(response.status_code, 200) self.assertEqual(response.all_content, bytes)
def get(url): if not Log: _late_import() """ USE json.net CONVENTIONS TO LINK TO INLINE OTHER JSON """ if url.find("://") == -1: Log.error("{{url}} must have a prototcol (eg http://) declared", url=url) if url.startswith("file://") and url[7] != "/": # RELATIVE if os.sep == "\\": url = "file:///" + os.getcwd().replace(os.sep, "/") + "/" + url[7:] else: url = "file://" + os.getcwd() + "/" + url[7:] if url[url.find("://") + 3] != "/": Log.error("{{url}} must be absolute", url=url) doc = wrap({"$ref": url}) phase1 = _replace_ref(doc, URL("")) # BLANK URL ONLY WORKS IF url IS ABSOLUTE phase2 = _replace_locals(phase1, [phase1]) return wrap(phase2)