def _replace_locals(node, doc_path): if is_data(node): # RECURS, DEEP COPY ref = None output = {} for k, v in node.items(): if k == "$ref": ref = v elif k == "$concat": if not is_sequence(v): Log.error("$concat expects an array of strings") return coalesce(node.get("separator"), "").join(v) elif v == None: continue else: output[k] = _replace_locals(v, [v] + doc_path) if not ref: return output # REFER TO SELF frag = ref.fragment if frag[0] == ".": # RELATIVE for i, p in enumerate(frag): if p != ".": if i > len(doc_path): Log.error( "{{frag|quote}} reaches up past the root document", frag=frag) new_value = get_attr(doc_path[i - 1], frag[i::]) break else: new_value = doc_path[len(frag) - 1] else: # ABSOLUTE new_value = get_attr(doc_path[-1], frag) new_value = _replace_locals(new_value, [new_value] + doc_path) if not output: return new_value # OPTIMIZATION FOR CASE WHEN node IS {} else: return unwrap(set_default(output, new_value)) elif is_list(node): candidate = [_replace_locals(n, [n] + doc_path) for n in node] # if all(p[0] is p[1] for p in zip(candidate, node)): # return node return candidate return node
def _replace_locals(node, doc_path): if isinstance(node, Mapping): # RECURS, DEEP COPY ref = None output = {} for k, v in node.items(): if k == "$ref": ref = v elif v == None: continue else: output[k] = _replace_locals(v, [v] + doc_path) if not ref: return output # REFER TO SELF frag = ref.fragment if frag[0] == ".": # RELATIVE for i, p in enumerate(frag): if p != ".": if i > len(doc_path): Log.error( "{{frag|quote}} reaches up past the root document", frag=frag) new_value = mo_dots.get_attr(doc_path[i - 1], frag[i::]) break else: new_value = doc_path[len(frag) - 1] else: # ABSOLUTE new_value = mo_dots.get_attr(doc_path[-1], frag) if new_value in doc_path: Log.error("encountered referential loop {{path|json}}", path=[new_value] + doc_path) new_value = _replace_locals(new_value, [new_value] + doc_path) if not output: return new_value # OPTIMIZATION FOR CASE WHEN node IS {} else: return unwrap(set_default(output, new_value)) elif isinstance(node, list): candidate = [_replace_locals(n, [n] + doc_path) for n in node] # if all(p[0] is p[1] for p in zip(candidate, node)): # return node return candidate return node
def update(self, command): try: command = wrap(command) eq = command.where.eq if eq.es_index: columns = self.find(eq.es_index, eq.name) columns = [ c for c in columns if all( get_attr(c, k) == v for k, v in eq.items()) ] else: columns = list(self) columns = jx.filter(columns, command.where) for col in list(columns): for k in command["clear"]: if k == ".": columns.remove(col) else: col[k] = None for k, v in command.set.items(): col[k] = v except Exception as e: Log.error("should not happen", cause=e)
def _replace_ref(node, url): if url.path.endswith("/"): url.path = url.path[:-1] if isinstance(node, Mapping): ref = None output = {} for k, v in node.items(): if k == "$ref": ref = URL(v) else: output[k] = _replace_ref(v, url) if not ref: return output node = output if not ref.scheme and not ref.path: # DO NOT TOUCH LOCAL REF YET output["$ref"] = ref return output if not ref.scheme: # SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH # REQUIRES THE CURRENT DOCUMENT'S SCHEME ref.scheme = url.scheme # FIND THE SCHEME AND LOAD IT if ref.scheme in scheme_loaders: new_value = scheme_loaders[ref.scheme](ref, url) else: raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme) if ref.fragment: new_value = mo_dots.get_attr(new_value, ref.fragment) DEBUG and Log.note( "Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value) if not output: output = new_value elif isinstance(output, text_type): Log.error("Can not handle set_default({{output}},{{new_value}})", output=output, new_value=new_value) else: output = unwrap(set_default(output, new_value)) DEBUG and Log.note("Return {{output}}", output=output) return output elif isinstance(node, list): output = [_replace_ref(n, url) for n in node] # if all(p[0] is p[1] for p in zip(output, node)): # return node return output return node
def _replace_locals(node, doc_path): if isinstance(node, Mapping): # RECURS, DEEP COPY ref = None output = {} for k, v in node.items(): if k == "$ref": ref = v elif v == None: continue else: output[k] = _replace_locals(v, [v] + doc_path) if not ref: return output # REFER TO SELF frag = ref.fragment if frag[0] == ".": # RELATIVE for i, p in enumerate(frag): if p != ".": if i>len(doc_path): Log.error("{{frag|quote}} reaches up past the root document", frag=frag) new_value = mo_dots.get_attr(doc_path[i-1], frag[i::]) break else: new_value = doc_path[len(frag) - 1] else: # ABSOLUTE new_value = mo_dots.get_attr(doc_path[-1], frag) new_value = _replace_locals(new_value, [new_value] + doc_path) if not output: return new_value # OPTIMIZATION FOR CASE WHEN node IS {} else: return unwrap(set_default(output, new_value)) elif isinstance(node, list): candidate = [_replace_locals(n, [n] + doc_path) for n in node] # if all(p[0] is p[1] for p in zip(candidate, node)): # return node return candidate return node
def _replace_ref(node, url): if url.path.endswith("/"): url.path = url.path[:-1] if isinstance(node, Mapping): ref = None output = {} for k, v in node.items(): if k == "$ref": ref = URL(v) else: output[k] = _replace_ref(v, url) if not ref: return output node = output if not ref.scheme and not ref.path: # DO NOT TOUCH LOCAL REF YET output["$ref"] = ref return output if not ref.scheme: # SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH # REQUIRES THE CURRENT DOCUMENT'S SCHEME ref.scheme = url.scheme # FIND THE SCHEME AND LOAD IT if ref.scheme in scheme_loaders: new_value = scheme_loaders[ref.scheme](ref, url) else: raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme) if ref.fragment: new_value = mo_dots.get_attr(new_value, ref.fragment) DEBUG and Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value) if not output: output = new_value elif isinstance(output, text_type): Log.error("Can not handle set_default({{output}},{{new_value}})", output=output, new_value=new_value) else: output = unwrap(set_default(output, new_value)) DEBUG and Log.note("Return {{output}}", output=output) return output elif isinstance(node, list): output = [_replace_ref(n, url) for n in node] # if all(p[0] is p[1] for p in zip(output, node)): # return node return output return node
def value2key(keys, val): if len(keys) == 1: if is_data(val): return get_attr(val, keys[0]), elif is_sequence(val): return val[0], return val, else: if is_data(val): return tuple(val[k] for k in keys) elif is_sequence(val): return tuple(val) else: Log.error("do not know what to do here")
def value2key(keys, val): if len(keys) == 1: if isinstance(val, Mapping): return get_attr(val, keys[0]), elif isinstance(val, (list, tuple)): return val[0], return val, else: if isinstance(val, Mapping): return tuple(val[k] for k in keys) elif isinstance(val, (list, tuple)): return tuple(val) else: Log.error("do not know what to do here")
def update(self, command): try: command = wrap(command) eq = command.where.eq if eq.es_index: columns = self.find(eq.es_index, eq.name) columns = [c for c in columns if all(get_attr(c, k) == v for k, v in eq.items())] else: columns = list(self) columns = jx.filter(columns, command.where) for col in columns: for k in command["clear"]: col[k] = None for k, v in command.set.items(): col[k] = v except Exception as e: Log.error("sould not happen", cause=e)
def assertAlmostEqual(test, expected, digits=None, places=None, msg=None, delta=None): show_detail = True test = unwrap(test) expected = unwrap(expected) try: if test is None and expected is None: return elif isinstance(test, UniqueIndex): if test ^ expected: Log.error("Sets do not match") elif isinstance(expected, Mapping) and isinstance(test, Mapping): for k, v2 in unwrap(expected).items(): v1 = test.get(k) assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta) elif isinstance(expected, Mapping): for k, v2 in expected.items(): if isinstance(k, basestring): v1 = mo_dots.get_attr(test, literal_field(k)) else: v1 = test[k] assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta) elif isinstance(test, (set, list)) and isinstance(expected, set): test = set(test) if len(test) != len(expected): Log.error( "Sets do not match, element count different:\n{{test|json|indent}}\nexpecting{{expectedtest|json|indent}}", test=test, expected=expected ) for e in expected: for t in test: try: assertAlmostEqual(t, e, msg=msg, digits=digits, places=places, delta=delta) break except Exception, _: pass else: Log.error("Sets do not match. {{value|json}} not found in {{test|json}}", value=e, test=test) elif isinstance(expected, types.FunctionType): return expected(test)
def _replace_ref(node, url): if url.path.endswith("/"): url.path = url.path[:-1] if is_data(node): refs = None output = {} for k, v in node.items(): if k == "$ref": refs = URL(v) else: output[k] = _replace_ref(v, url) if not refs: return output ref_found = False ref_error = None ref_remain = [] for ref in listwrap(refs): if not ref.scheme and not ref.path: # DO NOT TOUCH LOCAL REF YET ref_remain.append(ref) ref_found = True continue if not ref.scheme: # SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH # REQUIRES THE CURRENT DOCUMENT'S SCHEME ref.scheme = url.scheme # FIND THE SCHEME AND LOAD IT if ref.scheme not in scheme_loaders: raise Log.error("unknown protocol {{scheme}}", scheme=ref.scheme) try: new_value = scheme_loaders[ref.scheme](ref, url) ref_found = True except Exception as e: ref_error = e continue if ref.fragment: new_value = get_attr(new_value, ref.fragment) DEBUG and Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value) if not output: output = new_value elif is_text(output): pass # WE HAVE A VALUE else: set_default(output, new_value) if not ref_found: raise ref_error if ref_remain: output["$ref"] = unwraplist(ref_remain) DEBUG and Log.note("Return {{output}}", output=output) return output elif is_list(node): output = [_replace_ref(n, url) for n in node] # if all(p[0] is p[1] for p in zip(output, node)): # return node return output return node
def __getattr__(self, item): obj = _get(self, "_obj") output = get_attr(obj, item) return datawrap(output)
def __getitem__(self, item): obj = _get(self, "_obj") output = get_attr(obj, item) return datawrap(output)
def __getitem__(self, item): obj = _get(self, OBJ) output = get_attr(obj, item) return datawrap(output)
def assertAlmostEqual(test, expected, digits=None, places=None, msg=None, delta=None): show_detail = True test = unwrap(test) expected = unwrap(expected) try: if test is None and expected is None: return elif isinstance(test, UniqueIndex): if test ^ expected: Log.error("Sets do not match") elif isinstance(expected, Mapping) and isinstance(test, Mapping): for k, v2 in unwrap(expected).items(): v1 = test.get(k) assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta) elif isinstance(expected, Mapping): for k, v2 in expected.items(): if isinstance(k, basestring): v1 = mo_dots.get_attr(test, literal_field(k)) else: v1 = test[k] assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta) elif isinstance(test, (set, list)) and isinstance(expected, set): test = set(test) if len(test) != len(expected): Log.error( "Sets do not match, element count different:\n{{test|json|indent}}\nexpecting{{expectedtest|json|indent}}", test=test, expected=expected) for e in expected: for t in test: try: assertAlmostEqual(t, e, msg=msg, digits=digits, places=places, delta=delta) break except Exception, _: pass else: Log.error( "Sets do not match. {{value|json}} not found in {{test|json}}", value=e, test=test) elif isinstance(expected, types.FunctionType): return expected(test)
def assertAlmostEqual(test, expected, digits=None, places=None, msg=None, delta=None): show_detail = True test = unwrap(test) expected = unwrap(expected) try: if test is None and expected is None: return elif test is expected: return elif is_text(expected): assertAlmostEqualValue(test, expected, msg=msg, digits=digits, places=places, delta=delta) elif isinstance(test, UniqueIndex): if test ^ expected: Log.error("Sets do not match") elif is_data(expected) and is_data(test): for k, v2 in unwrap(expected).items(): v1 = test.get(k) assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta) elif is_data(expected): for k, v2 in expected.items(): if is_text(k): v1 = mo_dots.get_attr(test, literal_field(k)) else: v1 = test[k] assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta) elif is_container(test) and isinstance(expected, set): test = set(wrap(t) for t in test) if len(test) != len(expected): Log.error( "Sets do not match, element count different:\n{{test|json|indent}}\nexpecting{{expectedtest|json|indent}}", test=test, expected=expected ) for e in expected: for t in test: try: assertAlmostEqual(t, e, msg=msg, digits=digits, places=places, delta=delta) break except Exception as _: pass else: Log.error("Sets do not match. {{value|json}} not found in {{test|json}}", value=e, test=test) elif isinstance(expected, types.FunctionType): return expected(test) elif hasattr(test, "__iter__") and hasattr(expected, "__iter__"): if test.__class__.__name__ == "ndarray": # numpy test = test.tolist() elif test.__class__.__name__ == "DataFrame": # pandas test = test[test.columns[0]].values.tolist() elif test.__class__.__name__ == "Series": # pandas test = test.values.tolist() if not expected and test == None: return if expected == None: expected = [] # REPRESENT NOTHING for a, b in zip_longest(test, expected): assertAlmostEqual(a, b, msg=msg, digits=digits, places=places, delta=delta) else: assertAlmostEqualValue(test, expected, msg=msg, digits=digits, places=places, delta=delta) except Exception as e: Log.error( "{{test|json|limit(10000)}} does not match expected {{expected|json|limit(10000)}}", test=test if show_detail else "[can not show]", expected=expected if show_detail else "[can not show]", cause=e )
def __getattr__(self, item): obj = _get(self, OBJ) output = get_attr(obj, item) return datawrap(output)