def _replace_locals(node, doc_path):
if isinstance(node, Mapping):
# RECURS, DEEP COPY
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = v
elif v == None:
continue
else:
output[k] = _replace_locals(v, [v] + doc_path)
if not ref:
return output
# REFER TO SELF
frag = ref.fragment
if frag[0] == ".":
# RELATIVE
for i, p in enumerate(frag):
if p != ".":
if i > len(doc_path):
_Log.error(
"{{frag|quote}} reaches up past the root document",
frag=frag)
new_value = dot.get_attr(doc_path[i - 1], frag[i::])
break
else:
new_value = doc_path[len(frag) - 1]
else:
# ABSOLUTE
new_value = dot.get_attr(doc_path[-1], frag)
new_value = _replace_locals(new_value, [new_value] + doc_path)
if not output:
return new_value # OPTIMIZATION FOR CASE WHEN node IS {}
else:
return unwrap(set_default(output, new_value))
elif isinstance(node, list):
candidate = [_replace_locals(n, [n] + doc_path) for n in node]
# if all(p[0] is p[1] for p in zip(candidate, node)):
# return node
return candidate
return node
def assertAlmostEqual(test, expected, digits=None, places=None, msg=None, delta=None):
show_detail=True
try:
if test==None and expected==None:
return
elif isinstance(test, UniqueIndex):
if test ^ expected:
Log.error("Sets do not match")
elif isinstance(expected, Mapping):
for k, v2 in expected.items():
if isinstance(k, basestring):
v1 = dot.get_attr(test, literal_field(k))
else:
show_deta =False
v1 = test[k]
assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta)
elif isinstance(test, set) and isinstance(expected, set):
if test ^ expected:
Log.error("Sets do not match")
elif isinstance(expected, types.FunctionType):
return expected(test)
elif hasattr(test, "__iter__") and hasattr(expected, "__iter__"):
for a, b in zipall(test, expected):
assertAlmostEqual(a, b, msg=msg, digits=digits, places=places, delta=delta)
else:
assertAlmostEqualValue(test, expected, msg=msg, digits=digits, places=places, delta=delta)
except Exception, e:
Log.error(
"{{test|json}} does not match expected {{expected|json}}",
test=test if show_detail else "[can not show]",
expected=expected if show_detail else "[can not show]",
cause=e
)
def _replace_locals(node, doc_path):
if isinstance(node, Mapping):
# RECURS, DEEP COPY
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = v
elif v == None:
continue
else:
output[k] = _replace_locals(v, [v] + doc_path)
if not ref:
return output
# REFER TO SELF
frag = ref.fragment
if frag[0] == ".":
# RELATIVE
for i, p in enumerate(frag):
if p != ".":
if i>len(doc_path):
_Log.error("{{frag|quote}} reaches up past the root document", frag=frag)
new_value = dot.get_attr(doc_path[i-1], frag[i::])
break
else:
new_value = doc_path[len(frag) - 1]
else:
# ABSOLUTE
new_value = dot.get_attr(doc_path[-1], frag)
new_value = _replace_locals(new_value, [new_value] + doc_path)
if not output:
return new_value # OPTIMIZATION FOR CASE WHEN node IS {}
else:
return unwrap(set_default(output, new_value))
elif isinstance(node, list):
candidate = [_replace_locals(n, [n] + doc_path) for n in node]
# if all(p[0] is p[1] for p in zip(candidate, node)):
# return node
return candidate
return node
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise _Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = dot.get_attr(new_value, ref.fragment)
if DEBUG:
_Log.note("Replace {{ref}} with {{new_value}}",
ref=ref,
new_value=new_value)
if not output:
output = new_value
else:
output = unwrap(set_default(output, new_value))
if DEBUG:
_Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise _Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = dot.get_attr(new_value, ref.fragment)
if DEBUG:
_Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value)
if not output:
output = new_value
else:
output = unwrap(set_default(output, new_value))
if DEBUG:
_Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def assertAlmostEqual(test,
expected,
digits=None,
places=None,
msg=None,
delta=None):
show_detail = True
try:
if test == None and expected == None:
return
elif isinstance(test, UniqueIndex):
if test ^ expected:
Log.error("Sets do not match")
elif isinstance(expected, Mapping):
for k, v2 in expected.items():
if isinstance(k, basestring):
v1 = dot.get_attr(test, literal_field(k))
else:
show_deta = False
v1 = test[k]
assertAlmostEqual(v1,
v2,
msg=msg,
digits=digits,
places=places,
delta=delta)
elif isinstance(test, (set, list)) and isinstance(expected, set):
test = set(test)
if len(test) != len(expected):
Log.error(
"Sets do not match, element count different:\n{{test|json|indent}}\nexpecting{{expectedtest|json|indent}}",
test=test,
expected=expected)
for e in expected:
for t in test:
try:
assertAlmostEqual(t,
e,
msg=msg,
digits=digits,
places=places,
delta=delta)
break
except Exception, _:
pass
else:
Log.error(
"Sets do not match. {{value|json}} not found in {{test|json}}",
value=e,
test=test)
elif isinstance(expected, types.FunctionType):
return expected(test)
def assertAlmostEqual(test,
expected,
digits=None,
places=None,
msg=None,
delta=None):
show_detail = True
try:
if test == None and expected == None:
return
elif isinstance(test, UniqueIndex):
if test ^ expected:
Log.error("Sets do not match")
elif isinstance(expected, Mapping):
for k, v2 in expected.items():
if isinstance(k, basestring):
v1 = dot.get_attr(test, literal_field(k))
else:
show_deta = False
v1 = test[k]
assertAlmostEqual(v1,
v2,
msg=msg,
digits=digits,
places=places,
delta=delta)
elif isinstance(test, set) and isinstance(expected, set):
if test ^ expected:
Log.error("Sets do not match")
elif isinstance(expected, types.FunctionType):
return expected(test)
elif hasattr(test, "__iter__") and hasattr(expected, "__iter__"):
for a, b in zipall(test, expected):
assertAlmostEqual(a,
b,
msg=msg,
digits=digits,
places=places,
delta=delta)
else:
assertAlmostEqualValue(test,
expected,
msg=msg,
digits=digits,
places=places,
delta=delta)
except Exception, e:
Log.error("{{test|json}} does not match expected {{expected|json}}",
test=test if show_detail else "[can not show]",
expected=expected if show_detail else "[can not show]",
cause=e)
def __init__(self,
name,
work_queue,
workers,
resources,
please_stop,
wait_forever=False,
settings=None):
# FIND THE WORKERS METHODS
settings.workers = []
for w in workers:
w = deepcopy(w)
for existing_worker in settings.workers:
try:
fuzzytestcase.assertAlmostEqual(existing_worker.source,
w.source)
fuzzytestcase.assertAlmostEqual(
existing_worker.transformer, w.transformer)
# SAME SOURCE AND TRANSFORMER, MERGE THE destinations
except Exception, e:
continue
destination = get_container(w.destination)
existing_worker._destination = Split(
existing_worker._destination, destination)
break
else:
t_name = w.transformer
w._transformer = dot.get_attr(sys.modules, t_name)
if not w._transformer:
Log.error(
"Can not find {{path}} to transformer (are you sure you are pointing to a function?)",
path=t_name)
w._source = get_container(w.source)
w._destination = get_container(w.destination)
settings.workers.append(w)
w._notify = []
for notify in listwrap(w.notify):
w._notify.append(aws.Queue(notify))
def __init__(
self,
name,
work_queue,
workers,
resources,
please_stop,
wait_forever=False,
settings=None
):
# FIND THE WORKERS METHODS
settings.workers = []
for w in workers:
w = deepcopy(w)
for existing_worker in settings.workers:
try:
fuzzytestcase.assertAlmostEqual(existing_worker.source, w.source)
fuzzytestcase.assertAlmostEqual(existing_worker.transformer, w.transformer)
# SAME SOURCE AND TRANSFORMER, MERGE THE destinations
except Exception, e:
continue
destination = get_container(w.destination)
existing_worker._destination = Split(existing_worker._destination, destination)
break
else:
t_name = w.transformer
w._transformer = dot.get_attr(sys.modules, t_name)
if not w._transformer:
Log.error("Can not find {{path}} to transformer (are you sure you are pointing to a function?)", path=t_name)
w._source = get_container(w.source)
w._destination = get_container(w.destination)
settings.workers.append(w)
w._notify = []
for notify in listwrap(w.notify):
w._notify.append(aws.Queue(notify))
def assertAlmostEqual(test, expected, digits=None, places=None, msg=None, delta=None):
show_detail=True
try:
if test==None and expected==None:
return
elif isinstance(test, UniqueIndex):
if test ^ expected:
Log.error("Sets do not match")
elif isinstance(expected, Mapping):
for k, v2 in expected.items():
if isinstance(k, basestring):
v1 = dot.get_attr(test, literal_field(k))
else:
show_deta =False
v1 = test[k]
assertAlmostEqual(v1, v2, msg=msg, digits=digits, places=places, delta=delta)
elif isinstance(test, (set, list)) and isinstance(expected, set):
test = set(test)
if len(test) != len(expected):
Log.error(
"Sets do not match, element count different:\n{{test|json|indent}}\nexpecting{{expectedtest|json|indent}}",
test=test,
expected=expected
)
for e in expected:
for t in test:
try:
assertAlmostEqual(t, e, msg=msg, digits=digits, places=places, delta=delta)
break
except Exception, _:
pass
else:
Log.error("Sets do not match. {{value|json}} not found in {{test|json}}", value=e, test=test)
elif isinstance(expected, types.FunctionType):
return expected(test)
def __getitem__(self, item):
obj = _get(self, "_obj")
output = get_attr(obj, item)
return dictwrap(output)
def __getitem__(self, item):
obj = _get(self, "_obj")
output = get_attr(obj, item)
return dictwrap(output)