def filter(data, where):
"""
where - a function that accepts (record, rownum, rows) and returns boolean
"""
if len(data) == 0 or where == None or where == TRUE:
return data
if isinstance(data, Container):
return data.filter(where)
if is_container(data):
temp = get(where)
dd = wrap(data)
return wrap(
[unwrap(d) for i, d in enumerate(data) if temp(wrap(d), i, dd)])
else:
Log.error("Do not know how to handle type {{type}}",
type=data.__class__.__name__)
try:
return drill_filter(where, data)
except Exception as _:
# WOW! THIS IS INEFFICIENT!
return wrap([
unwrap(d)
for d in drill_filter(where, [DataObject(d) for d in data])
])
def handle(self, record):
o = value2json(DataObject(record))
if record:
pass
if "this is a problem" not in e.args:
Log.error(
"We expect Python to, at least, report the first order problem"
)
if "this is the root cause" in e.args:
Log.error(
"We do not expect Python to report exception chains"
)
def _scrub(value, is_done):
type_ = value.__class__
if type_ in (NoneType, NullType):
return None
elif type_ is unicode:
value_ = value.strip()
if value_:
return value_
else:
return None
elif type_ is float:
if math.isnan(value) or math.isinf(value):
return None
return value
elif type_ in (int, long, bool):
return value
elif type_ in (date, datetime):
return float(datetime2unix(value))
elif type_ is timedelta:
return value.total_seconds()
elif type_ is Date:
return float(value.unix)
elif type_ is Duration:
return float(value.seconds)
elif type_ is str:
return utf82unicode(value)
elif type_ is Decimal:
return float(value)
elif type_ is Data:
return _scrub(_get(value, '_dict'), is_done)
elif isinstance(value, Mapping):
_id = id(value)
if _id in is_done:
Log.warning("possible loop in structure detected")
return '"<LOOP IN STRUCTURE>"'
is_done.add(_id)
output = {}
for k, v in value.iteritems():
if isinstance(k, basestring):
pass
elif hasattr(k, "__unicode__"):
k = unicode(k)
else:
Log.error("keys must be strings")
v = _scrub(v, is_done)
if v != None or isinstance(v, Mapping):
output[k] = v
is_done.discard(_id)
return output
elif type_ in (tuple, list, FlatList):
output = []
for v in value:
v = _scrub(v, is_done)
output.append(v)
return output
elif type_ is type:
return value.__name__
elif type_.__name__ == "bool_": # DEAR ME! Numpy has it's own booleans (value==False could be used, but 0==False in Python. DOH!)
if value == False:
return False
else:
return True
elif not isinstance(value, Except) and isinstance(value, Exception):
return _scrub(Except.wrap(value), is_done)
elif hasattr(value, '__data__'):
try:
return _scrub(value.__data__(), is_done)
except Exception as e:
Log.error("problem with calling __json__()", e)
elif hasattr(value, 'co_code') or hasattr(value, "f_locals"):
return None
elif hasattr(value, '__iter__'):
output = []
for v in value:
v = _scrub(v, is_done)
output.append(v)
return output
elif hasattr(value, '__call__'):
return repr(value)
else:
return _scrub(DataObject(value), is_done)
def _scrub(value, is_done, stack, scrub_text, scrub_number):
if FIND_LOOPS:
_id = id(value)
if _id in stack:
Log.error("loop in JSON")
stack = stack + [_id]
type_ = value.__class__
if type_ in (none_type, NullType):
return None
elif type_ is text_type:
return scrub_text(value)
elif type_ is float:
if math.isnan(value) or math.isinf(value):
return None
return scrub_number(value)
elif type_ is bool:
return value
elif type_ in integer_types:
return scrub_number(value)
elif type_ in (date, datetime):
return scrub_number(datetime2unix(value))
elif type_ is timedelta:
return value.total_seconds()
elif type_ is Date:
return scrub_number(value.unix)
elif type_ is Duration:
return scrub_number(value.seconds)
elif type_ is str:
return utf82unicode(value)
elif type_ is Decimal:
return scrub_number(value)
elif type_ is Data:
return _scrub(_get(value, SLOT), is_done, stack, scrub_text,
scrub_number)
elif is_data(value):
_id = id(value)
if _id in is_done:
Log.warning("possible loop in structure detected")
return '"<LOOP IN STRUCTURE>"'
is_done.add(_id)
output = {}
for k, v in value.items():
if is_text(k):
pass
elif is_binary(k):
k = k.decode('utf8')
# elif hasattr(k, "__unicode__"):
# k = text_type(k)
else:
Log.error("keys must be strings")
v = _scrub(v, is_done, stack, scrub_text, scrub_number)
if v != None or is_data(v):
output[k] = v
is_done.discard(_id)
return output
elif type_ in (tuple, list, FlatList):
output = []
for v in value:
v = _scrub(v, is_done, stack, scrub_text, scrub_number)
output.append(v)
return output # if output else None
elif type_ is type:
return value.__name__
elif type_.__name__ == "bool_": # DEAR ME! Numpy has it's own booleans (value==False could be used, but 0==False in Python. DOH!)
if value == False:
return False
else:
return True
elif not isinstance(value, Except) and isinstance(value, Exception):
return _scrub(Except.wrap(value), is_done, stack, scrub_text,
scrub_number)
elif hasattr(value, '__data__'):
try:
return _scrub(value.__data__(), is_done, stack, scrub_text,
scrub_number)
except Exception as e:
Log.error("problem with calling __json__()", e)
elif hasattr(value, 'co_code') or hasattr(value, "f_locals"):
return None
elif hasattr(value, '__iter__'):
output = []
for v in value:
v = _scrub(v, is_done, stack, scrub_text, scrub_number)
output.append(v)
return output
elif hasattr(value, '__call__'):
return text_type(repr(value))
else:
return _scrub(DataObject(value), is_done, stack, scrub_text,
scrub_number)
