def filter(data, where): """ where - a function that accepts (record, rownum, rows) and returns boolean """ if len(data) == 0 or where == None or where == TRUE: return data if isinstance(data, Container): return data.filter(where) if is_container(data): temp = get(where) dd = wrap(data) return wrap( [unwrap(d) for i, d in enumerate(data) if temp(wrap(d), i, dd)]) else: Log.error("Do not know how to handle type {{type}}", type=data.__class__.__name__) try: return drill_filter(where, data) except Exception as _: # WOW! THIS IS INEFFICIENT! return wrap([ unwrap(d) for d in drill_filter(where, [DataObject(d) for d in data]) ])
def handle(self, record): o = value2json(DataObject(record)) if record: pass if "this is a problem" not in e.args: Log.error( "We expect Python to, at least, report the first order problem" ) if "this is the root cause" in e.args: Log.error( "We do not expect Python to report exception chains" )
def _scrub(value, is_done): type_ = value.__class__ if type_ in (NoneType, NullType): return None elif type_ is unicode: value_ = value.strip() if value_: return value_ else: return None elif type_ is float: if math.isnan(value) or math.isinf(value): return None return value elif type_ in (int, long, bool): return value elif type_ in (date, datetime): return float(datetime2unix(value)) elif type_ is timedelta: return value.total_seconds() elif type_ is Date: return float(value.unix) elif type_ is Duration: return float(value.seconds) elif type_ is str: return utf82unicode(value) elif type_ is Decimal: return float(value) elif type_ is Data: return _scrub(_get(value, '_dict'), is_done) elif isinstance(value, Mapping): _id = id(value) if _id in is_done: Log.warning("possible loop in structure detected") return '"<LOOP IN STRUCTURE>"' is_done.add(_id) output = {} for k, v in value.iteritems(): if isinstance(k, basestring): pass elif hasattr(k, "__unicode__"): k = unicode(k) else: Log.error("keys must be strings") v = _scrub(v, is_done) if v != None or isinstance(v, Mapping): output[k] = v is_done.discard(_id) return output elif type_ in (tuple, list, FlatList): output = [] for v in value: v = _scrub(v, is_done) output.append(v) return output elif type_ is type: return value.__name__ elif type_.__name__ == "bool_": # DEAR ME! Numpy has it's own booleans (value==False could be used, but 0==False in Python. DOH!) if value == False: return False else: return True elif not isinstance(value, Except) and isinstance(value, Exception): return _scrub(Except.wrap(value), is_done) elif hasattr(value, '__data__'): try: return _scrub(value.__data__(), is_done) except Exception as e: Log.error("problem with calling __json__()", e) elif hasattr(value, 'co_code') or hasattr(value, "f_locals"): return None elif hasattr(value, '__iter__'): output = [] for v in value: v = _scrub(v, is_done) output.append(v) return output elif hasattr(value, '__call__'): return repr(value) else: return _scrub(DataObject(value), is_done)
def _scrub(value, is_done, stack, scrub_text, scrub_number): if FIND_LOOPS: _id = id(value) if _id in stack: Log.error("loop in JSON") stack = stack + [_id] type_ = value.__class__ if type_ in (none_type, NullType): return None elif type_ is text_type: return scrub_text(value) elif type_ is float: if math.isnan(value) or math.isinf(value): return None return scrub_number(value) elif type_ is bool: return value elif type_ in integer_types: return scrub_number(value) elif type_ in (date, datetime): return scrub_number(datetime2unix(value)) elif type_ is timedelta: return value.total_seconds() elif type_ is Date: return scrub_number(value.unix) elif type_ is Duration: return scrub_number(value.seconds) elif type_ is str: return utf82unicode(value) elif type_ is Decimal: return scrub_number(value) elif type_ is Data: return _scrub(_get(value, SLOT), is_done, stack, scrub_text, scrub_number) elif is_data(value): _id = id(value) if _id in is_done: Log.warning("possible loop in structure detected") return '"<LOOP IN STRUCTURE>"' is_done.add(_id) output = {} for k, v in value.items(): if is_text(k): pass elif is_binary(k): k = k.decode('utf8') # elif hasattr(k, "__unicode__"): # k = text_type(k) else: Log.error("keys must be strings") v = _scrub(v, is_done, stack, scrub_text, scrub_number) if v != None or is_data(v): output[k] = v is_done.discard(_id) return output elif type_ in (tuple, list, FlatList): output = [] for v in value: v = _scrub(v, is_done, stack, scrub_text, scrub_number) output.append(v) return output # if output else None elif type_ is type: return value.__name__ elif type_.__name__ == "bool_": # DEAR ME! Numpy has it's own booleans (value==False could be used, but 0==False in Python. DOH!) if value == False: return False else: return True elif not isinstance(value, Except) and isinstance(value, Exception): return _scrub(Except.wrap(value), is_done, stack, scrub_text, scrub_number) elif hasattr(value, '__data__'): try: return _scrub(value.__data__(), is_done, stack, scrub_text, scrub_number) except Exception as e: Log.error("problem with calling __json__()", e) elif hasattr(value, 'co_code') or hasattr(value, "f_locals"): return None elif hasattr(value, '__iter__'): output = [] for v in value: v = _scrub(v, is_done, stack, scrub_text, scrub_number) output.append(v) return output elif hasattr(value, '__call__'): return text_type(repr(value)) else: return _scrub(DataObject(value), is_done, stack, scrub_text, scrub_number)