def _replace_locals(node, doc_path):
if is_data(node):
# RECURS, DEEP COPY
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = v
elif k == "$concat":
if not is_sequence(v):
Log.error("$concat expects an array of strings")
return coalesce(node.get("separator"), "").join(v)
elif v == None:
continue
else:
output[k] = _replace_locals(v, [v] + doc_path)
if not ref:
return output
# REFER TO SELF
frag = ref.fragment
if frag[0] == ".":
# RELATIVE
for i, p in enumerate(frag):
if p != ".":
if i > len(doc_path):
Log.error(
"{{frag|quote}} reaches up past the root document",
frag=frag)
new_value = get_attr(doc_path[i - 1], frag[i::])
break
else:
new_value = doc_path[len(frag) - 1]
else:
# ABSOLUTE
new_value = get_attr(doc_path[-1], frag)
new_value = _replace_locals(new_value, [new_value] + doc_path)
if not output:
return new_value # OPTIMIZATION FOR CASE WHEN node IS {}
else:
return unwrap(set_default(output, new_value))
elif is_list(node):
candidate = [_replace_locals(n, [n] + doc_path) for n in node]
# if all(p[0] is p[1] for p in zip(candidate, node)):
# return node
return candidate
return node
def _replace_locals(node, doc_path):
if isinstance(node, Mapping):
# RECURS, DEEP COPY
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = v
elif v == None:
continue
else:
output[k] = _replace_locals(v, [v] + doc_path)
if not ref:
return output
# REFER TO SELF
frag = ref.fragment
if frag[0] == ".":
# RELATIVE
for i, p in enumerate(frag):
if p != ".":
if i > len(doc_path):
Log.error(
"{{frag|quote}} reaches up past the root document",
frag=frag)
new_value = mo_dots.get_attr(doc_path[i - 1], frag[i::])
break
else:
new_value = doc_path[len(frag) - 1]
else:
# ABSOLUTE
new_value = mo_dots.get_attr(doc_path[-1], frag)
if new_value in doc_path:
Log.error("encountered referential loop {{path|json}}",
path=[new_value] + doc_path)
new_value = _replace_locals(new_value, [new_value] + doc_path)
if not output:
return new_value # OPTIMIZATION FOR CASE WHEN node IS {}
else:
return unwrap(set_default(output, new_value))
elif isinstance(node, list):
candidate = [_replace_locals(n, [n] + doc_path) for n in node]
# if all(p[0] is p[1] for p in zip(candidate, node)):
# return node
return candidate
return node
def update(self, command):
try:
command = wrap(command)
eq = command.where.eq
if eq.es_index:
columns = self.find(eq.es_index, eq.name)
columns = [
c for c in columns if all(
get_attr(c, k) == v for k, v in eq.items())
]
else:
columns = list(self)
columns = jx.filter(columns, command.where)
for col in list(columns):
for k in command["clear"]:
if k == ".":
columns.remove(col)
else:
col[k] = None
for k, v in command.set.items():
col[k] = v
except Exception as e:
Log.error("should not happen", cause=e)
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = mo_dots.get_attr(new_value, ref.fragment)
DEBUG and Log.note(
"Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value)
if not output:
output = new_value
elif isinstance(output, text_type):
Log.error("Can not handle set_default({{output}},{{new_value}})",
output=output,
new_value=new_value)
else:
output = unwrap(set_default(output, new_value))
DEBUG and Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def _replace_locals(node, doc_path):
if isinstance(node, Mapping):
# RECURS, DEEP COPY
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = v
elif v == None:
continue
else:
output[k] = _replace_locals(v, [v] + doc_path)
if not ref:
return output
# REFER TO SELF
frag = ref.fragment
if frag[0] == ".":
# RELATIVE
for i, p in enumerate(frag):
if p != ".":
if i>len(doc_path):
Log.error("{{frag|quote}} reaches up past the root document", frag=frag)
new_value = mo_dots.get_attr(doc_path[i-1], frag[i::])
break
else:
new_value = doc_path[len(frag) - 1]
else:
# ABSOLUTE
new_value = mo_dots.get_attr(doc_path[-1], frag)
new_value = _replace_locals(new_value, [new_value] + doc_path)
if not output:
return new_value # OPTIMIZATION FOR CASE WHEN node IS {}
else:
return unwrap(set_default(output, new_value))
elif isinstance(node, list):
candidate = [_replace_locals(n, [n] + doc_path) for n in node]
# if all(p[0] is p[1] for p in zip(candidate, node)):
# return node
return candidate
return node
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = mo_dots.get_attr(new_value, ref.fragment)
DEBUG and Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value)
if not output:
output = new_value
elif isinstance(output, text_type):
Log.error("Can not handle set_default({{output}},{{new_value}})", output=output, new_value=new_value)
else:
output = unwrap(set_default(output, new_value))
DEBUG and Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def value2key(keys, val):
if len(keys) == 1:
if is_data(val):
return get_attr(val, keys[0]),
elif is_sequence(val):
return val[0],
return val,
else:
if is_data(val):
return tuple(val[k] for k in keys)
elif is_sequence(val):
return tuple(val)
else:
Log.error("do not know what to do here")
def value2key(keys, val):
if len(keys) == 1:
if isinstance(val, Mapping):
return get_attr(val, keys[0]),
elif isinstance(val, (list, tuple)):
return val[0],
return val,
else:
if isinstance(val, Mapping):
return tuple(val[k] for k in keys)
elif isinstance(val, (list, tuple)):
return tuple(val)
else:
Log.error("do not know what to do here")
def value2key(keys, val):
if len(keys) == 1:
if isinstance(val, Mapping):
return get_attr(val, keys[0]),
elif isinstance(val, (list, tuple)):
return val[0],
return val,
else:
if isinstance(val, Mapping):
return tuple(val[k] for k in keys)
elif isinstance(val, (list, tuple)):
return tuple(val)
else:
Log.error("do not know what to do here")
def value2key(keys, val):
if len(keys) == 1:
if is_data(val):
return get_attr(val, keys[0]),
elif is_sequence(val):
return val[0],
return val,
else:
if is_data(val):
return tuple(val[k] for k in keys)
elif is_sequence(val):
return tuple(val)
else:
Log.error("do not know what to do here")
def update(self, command):
try:
command = wrap(command)
eq = command.where.eq
if eq.es_index:
columns = self.find(eq.es_index, eq.name)
columns = [c for c in columns if all(get_attr(c, k) == v for k, v in eq.items())]
else:
columns = list(self)
columns = jx.filter(columns, command.where)
for col in columns:
for k in command["clear"]:
col[k] = None
for k, v in command.set.items():
col[k] = v
except Exception as e:
Log.error("sould not happen", cause=e)
def assertAlmostEqual(test, expected, digits=None, places=None, msg=None, delta=None):
show_detail = True
test = unwrap(test)
expected = unwrap(expected)
try:
if test is None and expected is None:
return
elif isinstance(test, UniqueIndex):
if test ^ expected:
Log.error("Sets do not match")
elif isinstance(expected, Mapping) and isinstance(test, Mapping):
for k, v2 in unwrap(expected).items():
v1 = test.get(k)
assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta)
elif isinstance(expected, Mapping):
for k, v2 in expected.items():
if isinstance(k, basestring):
v1 = mo_dots.get_attr(test, literal_field(k))
else:
v1 = test[k]
assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta)
elif isinstance(test, (set, list)) and isinstance(expected, set):
test = set(test)
if len(test) != len(expected):
Log.error(
"Sets do not match, element count different:\n{{test|json|indent}}\nexpecting{{expectedtest|json|indent}}",
test=test,
expected=expected
)
for e in expected:
for t in test:
try:
assertAlmostEqual(t, e, msg=msg, digits=digits, places=places, delta=delta)
break
except Exception, _:
pass
else:
Log.error("Sets do not match. {{value|json}} not found in {{test|json}}", value=e, test=test)
elif isinstance(expected, types.FunctionType):
return expected(test)
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if is_data(node):
refs = None
output = {}
for k, v in node.items():
if k == "$ref":
refs = URL(v)
else:
output[k] = _replace_ref(v, url)
if not refs:
return output
ref_found = False
ref_error = None
ref_remain = []
for ref in listwrap(refs):
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
ref_remain.append(ref)
ref_found = True
continue
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme not in scheme_loaders:
raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
try:
new_value = scheme_loaders[ref.scheme](ref, url)
ref_found = True
except Exception as e:
ref_error = e
continue
if ref.fragment:
new_value = get_attr(new_value, ref.fragment)
DEBUG and Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value)
if not output:
output = new_value
elif is_text(output):
pass # WE HAVE A VALUE
else:
set_default(output, new_value)
if not ref_found:
raise ref_error
if ref_remain:
output["$ref"] = unwraplist(ref_remain)
DEBUG and Log.note("Return {{output}}", output=output)
return output
elif is_list(node):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def __getattr__(self, item):
obj = _get(self, "_obj")
output = get_attr(obj, item)
return datawrap(output)
def __getitem__(self, item):
obj = _get(self, "_obj")
output = get_attr(obj, item)
return datawrap(output)
def __getitem__(self, item):
obj = _get(self, OBJ)
output = get_attr(obj, item)
return datawrap(output)
def assertAlmostEqual(test,
expected,
digits=None,
places=None,
msg=None,
delta=None):
show_detail = True
test = unwrap(test)
expected = unwrap(expected)
try:
if test is None and expected is None:
return
elif isinstance(test, UniqueIndex):
if test ^ expected:
Log.error("Sets do not match")
elif isinstance(expected, Mapping) and isinstance(test, Mapping):
for k, v2 in unwrap(expected).items():
v1 = test.get(k)
assertAlmostEqual(v1,
v2,
msg=msg,
digits=digits,
places=places,
delta=delta)
elif isinstance(expected, Mapping):
for k, v2 in expected.items():
if isinstance(k, basestring):
v1 = mo_dots.get_attr(test, literal_field(k))
else:
v1 = test[k]
assertAlmostEqual(v1,
v2,
msg=msg,
digits=digits,
places=places,
delta=delta)
elif isinstance(test, (set, list)) and isinstance(expected, set):
test = set(test)
if len(test) != len(expected):
Log.error(
"Sets do not match, element count different:\n{{test|json|indent}}\nexpecting{{expectedtest|json|indent}}",
test=test,
expected=expected)
for e in expected:
for t in test:
try:
assertAlmostEqual(t,
e,
msg=msg,
digits=digits,
places=places,
delta=delta)
break
except Exception, _:
pass
else:
Log.error(
"Sets do not match. {{value|json}} not found in {{test|json}}",
value=e,
test=test)
elif isinstance(expected, types.FunctionType):
return expected(test)
def assertAlmostEqual(test, expected, digits=None, places=None, msg=None, delta=None):
show_detail = True
test = unwrap(test)
expected = unwrap(expected)
try:
if test is None and expected is None:
return
elif test is expected:
return
elif is_text(expected):
assertAlmostEqualValue(test, expected, msg=msg, digits=digits, places=places, delta=delta)
elif isinstance(test, UniqueIndex):
if test ^ expected:
Log.error("Sets do not match")
elif is_data(expected) and is_data(test):
for k, v2 in unwrap(expected).items():
v1 = test.get(k)
assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta)
elif is_data(expected):
for k, v2 in expected.items():
if is_text(k):
v1 = mo_dots.get_attr(test, literal_field(k))
else:
v1 = test[k]
assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta)
elif is_container(test) and isinstance(expected, set):
test = set(wrap(t) for t in test)
if len(test) != len(expected):
Log.error(
"Sets do not match, element count different:\n{{test|json|indent}}\nexpecting{{expectedtest|json|indent}}",
test=test,
expected=expected
)
for e in expected:
for t in test:
try:
assertAlmostEqual(t, e, msg=msg, digits=digits, places=places, delta=delta)
break
except Exception as _:
pass
else:
Log.error("Sets do not match. {{value|json}} not found in {{test|json}}", value=e, test=test)
elif isinstance(expected, types.FunctionType):
return expected(test)
elif hasattr(test, "__iter__") and hasattr(expected, "__iter__"):
if test.__class__.__name__ == "ndarray": # numpy
test = test.tolist()
elif test.__class__.__name__ == "DataFrame": # pandas
test = test[test.columns[0]].values.tolist()
elif test.__class__.__name__ == "Series": # pandas
test = test.values.tolist()
if not expected and test == None:
return
if expected == None:
expected = [] # REPRESENT NOTHING
for a, b in zip_longest(test, expected):
assertAlmostEqual(a, b, msg=msg, digits=digits, places=places, delta=delta)
else:
assertAlmostEqualValue(test, expected, msg=msg, digits=digits, places=places, delta=delta)
except Exception as e:
Log.error(
"{{test|json|limit(10000)}} does not match expected {{expected|json|limit(10000)}}",
test=test if show_detail else "[can not show]",
expected=expected if show_detail else "[can not show]",
cause=e
)
def __getattr__(self, item):
obj = _get(self, OBJ)
output = get_attr(obj, item)
return datawrap(output)
