def get(url):
"""
USE json.net CONVENTIONS TO LINK TO INLINE OTHER JSON
"""
url = str(url)
if url.find("://") == -1:
Log.error("{{url}} must have a prototcol (eg http://) declared",
url=url)
base = URL("")
if url.startswith("file://") and url[7] != "/":
if os.sep == "\\":
base = URL("file:///" +
os.getcwd().replace(os.sep, "/").rstrip("/") + "/.")
else:
base = URL("file://" + os.getcwd().rstrip("/") + "/.")
elif url[url.find("://") + 3] != "/":
Log.error("{{url}} must be absolute", url=url)
phase1 = _replace_ref(wrap({"$ref": url}),
base) # BLANK URL ONLY WORKS IF url IS ABSOLUTE
try:
phase2 = _replace_locals(phase1, [phase1])
return wrap(phase2)
except Exception as e:
Log.error("problem replacing locals in\n{{phase1}}",
phase1=phase1,
cause=e)
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = mo_dots.get_attr(new_value, ref.fragment)
if DEBUG:
Log.note("Replace {{ref}} with {{new_value}}",
ref=ref,
new_value=new_value)
if not output:
output = new_value
else:
output = unwrap(set_default(output, new_value))
if DEBUG:
Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = mo_dots.get_attr(new_value, ref.fragment)
if DEBUG:
Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value)
if not output:
output = new_value
else:
output = unwrap(set_default(output, new_value))
if DEBUG:
Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def expand(doc, doc_url="param://", params=None):
"""
ASSUMING YOU ALREADY PULED THE doc FROM doc_url, YOU CAN STILL USE THE
EXPANDING FEATURE
:param doc: THE DATA STRUCTURE FROM JSON SOURCE
:param doc_url: THE URL THIS doc CAME FROM (DEFAULT USES params AS A DOCUMENT SOURCE)
:param params: EXTRA PARAMETERS NOT FOUND IN THE doc_url PARAMETERS (WILL SUPERSEDE PARAMETERS FROM doc_url)
:return: EXPANDED JSON-SERIALIZABLE STRUCTURE
"""
if doc_url.find("://") == -1:
Log.error("{{url}} must have a prototcol (eg http://) declared", url=doc_url)
url = URL(doc_url)
url.query = set_default(url.query, params)
phase1 = _replace_ref(doc, url) # BLANK URL ONLY WORKS IF url IS ABSOLUTE
phase2 = _replace_locals(phase1, [phase1])
return wrap(phase2)
def expand(doc, doc_url="param://", params=None):
"""
ASSUMING YOU ALREADY PULED THE doc FROM doc_url, YOU CAN STILL USE THE
EXPANDING FEATURE
:param doc: THE DATA STRUCTURE FROM JSON SOURCE
:param doc_url: THE URL THIS doc CAME FROM (DEFAULT USES params AS A DOCUMENT SOURCE)
:param params: EXTRA PARAMETERS NOT FOUND IN THE doc_url PARAMETERS (WILL SUPERSEDE PARAMETERS FROM doc_url)
:return: EXPANDED JSON-SERIALIZABLE STRUCTURE
"""
if doc_url.find("://") == -1:
Log.error("{{url}} must have a prototcol (eg http://) declared",
url=doc_url)
url = URL(doc_url)
url.query = set_default(url.query, params)
phase1 = _replace_ref(doc, url) # BLANK URL ONLY WORKS IF url IS ABSOLUTE
phase2 = _replace_locals(phase1, [phase1])
return wrap(phase2)
def _run_sql_query(self, sql):
if not self.utils.sql_url:
Log.error("This test requires a `sql_url` parameter in the settings file")
test = Data(data=simple_test_data)
self.utils.fill_container(test)
sql = sql.replace(TEST_TABLE, test.query['from'])
url = URL(self.utils.sql_url)
response = self.utils.post_till_response(str(url), json={"meta": {"testing": True}, "sql": sql})
self.assertEqual(response.status_code, 200)
return json2value(utf82unicode(response.all_content))
def new_instance(
host,
index,
type=None,
alias=None,
name=None,
port=9200,
read_only=True,
timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
wait_for_active_shards=1, # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
typed=None,
kwargs=None):
try:
known = known_hosts.get((host, port))
if known:
return known(kwargs=kwargs)
url = URL(host)
url.port = port
status = http.get_json(text_type(url), stream=False)
version = status.version.number
if version.startswith("1."):
from jx_elasticsearch.es14 import ES14
type2container.setdefault("elasticsearch", ES14)
known_hosts[(host, port)] = ES14
output = ES14(kwargs=kwargs)
return output
elif version.startswith(("5.", "6.")):
from jx_elasticsearch.es52 import ES52
type2container.setdefault("elasticsearch", ES52)
known_hosts[(host, port)] = ES52
output = ES52(kwargs=kwargs)
return output
else:
Log.error("No jx interpreter for Elasticsearch {{version}}",
version=version)
except Exception as e:
Log.error("Can not make an interpreter for Elasticsearch", cause=e)