def typed_encode(value, sub_schema, path, net_new_properties, buffer): """ :param value: THE DATA STRUCTURE TO ENCODE :param sub_schema: dict FROM PATH TO Column DESCRIBING THE TYPE :param path: list OF CURRENT PATH :param net_new_properties: list FOR ADDING NEW PROPERTIES NOT FOUND IN sub_schema :param buffer: UnicodeBuilder OBJECT :return: """ try: # from jx_base import Column if sub_schema.__class__.__name__=='Column': value_json_type = python_type_to_json_type[value.__class__] column_json_type = es_type_to_json_type[sub_schema.es_type] if value_json_type == column_json_type: pass # ok elif value_json_type == NESTED and all(python_type_to_json_type[v.__class__] == column_json_type for v in value if v != None): pass # empty arrays can be anything else: from mo_logs import Log Log.error("Can not store {{value}} in {{column|quote}}", value=value, column=sub_schema.name) sub_schema = {json_type_to_inserter_type[value_json_type]: sub_schema} if value == None: from mo_logs import Log Log.error("can not encode null (missing) values") elif value is True: if BOOLEAN_TYPE not in sub_schema: sub_schema[BOOLEAN_TYPE] = {} net_new_properties.append(path + [BOOLEAN_TYPE]) append(buffer, '{') append(buffer, QUOTED_BOOLEAN_TYPE) append(buffer, 'true}') return elif value is False: if BOOLEAN_TYPE not in sub_schema: sub_schema[BOOLEAN_TYPE] = {} net_new_properties.append(path + [BOOLEAN_TYPE]) append(buffer, '{') append(buffer, QUOTED_BOOLEAN_TYPE) append(buffer, 'false}') return _type = value.__class__ if _type in (dict, Data): if sub_schema.__class__.__name__ == 'Column': from mo_logs import Log Log.error("Can not handle {{column|json}}", column=sub_schema) if NESTED_TYPE in sub_schema: # PREFER NESTED, WHEN SEEN BEFORE if value: append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) append(buffer, '[') _dict2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, ']' + COMMA) append(buffer, QUOTED_EXISTS_TYPE) append(buffer, text_type(len(value))) append(buffer, '}') else: # SINGLETON LIST append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) append(buffer, '[{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}]') append(buffer, COMMA) append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}') else: if EXISTS_TYPE not in sub_schema: sub_schema[EXISTS_TYPE] = {} net_new_properties.append(path + [EXISTS_TYPE]) if value: _dict2json(value, sub_schema, path, net_new_properties, buffer) else: append(buffer, '{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}') elif _type is binary_type: if STRING_TYPE not in sub_schema: sub_schema[STRING_TYPE] = True net_new_properties.append(path + [STRING_TYPE]) append(buffer, '{') append(buffer, QUOTED_STRING_TYPE) append(buffer, '"') try: v = utf82unicode(value) except Exception as e: raise problem_serializing(value, e) for c in v: append(buffer, ESCAPE_DCT.get(c, c)) append(buffer, '"}') elif _type is text_type: if STRING_TYPE not in sub_schema: sub_schema[STRING_TYPE] = True net_new_properties.append(path + [STRING_TYPE]) append(buffer, '{') append(buffer, QUOTED_STRING_TYPE) append(buffer, '"') for c in value: append(buffer, ESCAPE_DCT.get(c, c)) append(buffer, '"}') elif _type in (int, long): if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, text_type(value)) append(buffer, '}') elif _type in (float, Decimal): if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value)) append(buffer, '}') elif _type in (set, list, tuple, FlatList): if len(value) == 0: append(buffer, '{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '0}') elif any(v.__class__ in (Data, dict, set, list, tuple, FlatList) for v in value): # THIS IS NOT DONE BECAUSE if len(value) == 1: if NESTED_TYPE in sub_schema: append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) _list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, '}') else: # NO NEED TO NEST, SO DO NOT DO IT typed_encode(value[0], sub_schema, path, net_new_properties, buffer) else: if NESTED_TYPE not in sub_schema: sub_schema[NESTED_TYPE] = {} net_new_properties.append(path + [NESTED_TYPE]) append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) _list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, '}') else: # ALLOW PRIMITIVE MULTIVALUES value = [v for v in value if v != None] types = list(set(json_type_to_inserter_type[python_type_to_json_type[v.__class__]] for v in value)) if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) append(buffer, '[]}') elif len(types) > 1: _list2json(value, sub_schema, path + [NESTED_TYPE], net_new_properties, buffer) else: element_type = types[0] if element_type not in sub_schema: sub_schema[element_type] = True net_new_properties.append(path + [element_type]) append(buffer, '{') append(buffer, quote(element_type)) append(buffer, COLON) _multivalue2json(value, sub_schema[element_type], path + [element_type], net_new_properties, buffer) append(buffer, '}') elif _type is date: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(time.mktime(value.timetuple()))) append(buffer, '}') elif _type is datetime: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(time.mktime(value.timetuple()))) append(buffer, '}') elif _type is Date: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value.unix)) append(buffer, '}') elif _type is timedelta: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value.total_seconds())) append(buffer, '}') elif _type is Duration: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value.seconds)) append(buffer, '}') elif _type is NullType: append(buffer, 'null') elif hasattr(value, '__data__'): typed_encode(value.__data__(), sub_schema, path, net_new_properties, buffer) elif hasattr(value, '__iter__'): if NESTED_TYPE not in sub_schema: sub_schema[NESTED_TYPE] = {} net_new_properties.append(path + [NESTED_TYPE]) append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) _iter2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, '}') else: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def _typed_encode(self, value, sub_schema, path, net_new_properties, _buffer): try: if isinstance(sub_schema, Column): value_json_type = python_type_to_json_type[value.__class__] column_json_type = es_type_to_json_type[sub_schema.es_type] if value_json_type == column_json_type: pass # ok elif value_json_type == NESTED and all( python_type_to_json_type[ v.__class__] == column_json_type for v in value if v != None): pass # empty arrays can be anything else: from mo_logs import Log Log.error("Can not store {{value}} in {{column|quote}}", value=value, column=sub_schema.names['.']) sub_schema = { json_type_to_inserter_type[value_json_type]: sub_schema } if value is None: append(_buffer, '{}') return elif value is True: if BOOLEAN_TYPE not in sub_schema: sub_schema[BOOLEAN_TYPE] = {} net_new_properties.append(path + [BOOLEAN_TYPE]) append(_buffer, '{' + QUOTED_BOOLEAN_TYPE + COLON + 'true}') return elif value is False: if BOOLEAN_TYPE not in sub_schema: sub_schema[BOOLEAN_TYPE] = {} net_new_properties.append(path + [BOOLEAN_TYPE]) append(_buffer, '{' + QUOTED_BOOLEAN_TYPE + COLON + 'false}') return _type = value.__class__ if _type in (dict, Data): if isinstance(sub_schema, Column): from mo_logs import Log Log.error("Can not handle {{column|json}}", column=sub_schema) if NESTED_TYPE in sub_schema: # PREFER NESTED, WHEN SEEN BEFORE if value: append(_buffer, '{' + QUOTED_NESTED_TYPE + COLON + '[') self._dict2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, _buffer) append( _buffer, ']' + COMMA + QUOTED_EXISTS_TYPE + COLON + text_type(len(value)) + '}') else: # SINGLETON LISTS OF null SHOULD NOT EXIST from mo_logs import Log Log.error("should not happen") else: if EXISTS_TYPE not in sub_schema: sub_schema[EXISTS_TYPE] = {} net_new_properties.append(path + [EXISTS_TYPE]) if value: self._dict2json(value, sub_schema, path, net_new_properties, _buffer) else: append(_buffer, '{' + QUOTED_EXISTS_TYPE + COLON + '0}') elif _type is binary_type: if STRING_TYPE not in sub_schema: sub_schema[STRING_TYPE] = True net_new_properties.append(path + [STRING_TYPE]) append(_buffer, '{' + QUOTED_STRING_TYPE + COLON + '"') try: v = utf82unicode(value) except Exception as e: raise problem_serializing(value, e) for c in v: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, '"}') elif _type is text_type: if STRING_TYPE not in sub_schema: sub_schema[STRING_TYPE] = True net_new_properties.append(path + [STRING_TYPE]) append(_buffer, '{' + QUOTED_STRING_TYPE + COLON + '"') for c in value: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, '"}') elif _type in (int, long, Decimal): if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON) append(_buffer, float2json(value)) append(_buffer, '}') elif _type is float: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON) append(_buffer, float2json(value)) append(_buffer, '}') elif _type in (set, list, tuple, FlatList): if len(value) == 0: append(_buffer, '{' + QUOTED_NESTED_TYPE + COLON + '[]}') elif any( isinstance(v, (Mapping, set, list, tuple, FlatList)) for v in value): if NESTED_TYPE not in sub_schema: sub_schema[NESTED_TYPE] = {} net_new_properties.append(path + [NESTED_TYPE]) append(_buffer, '{' + QUOTED_NESTED_TYPE + COLON) self._list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, _buffer) append(_buffer, '}') else: # ALLOW PRIMITIVE MULTIVALUES value = [v for v in value if v != None] types = list( set(python_type_to_json_type[v.__class__] for v in value)) if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM append(_buffer, '{' + QUOTED_NESTED_TYPE + COLON + '[]}') elif len(types) > 1: from mo_logs import Log Log.error("Can not handle multi-typed multivalues") else: element_type = json_type_to_inserter_type[types[0]] if element_type not in sub_schema: sub_schema[element_type] = True net_new_properties.append(path + [element_type]) append(_buffer, '{' + quote(element_type) + COLON) self._multivalue2json(value, sub_schema[element_type], path + [element_type], net_new_properties, _buffer) append(_buffer, '}') elif _type is date: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON) append(_buffer, float2json(time.mktime(value.timetuple()))) append(_buffer, '}') elif _type is datetime: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON) append(_buffer, float2json(time.mktime(value.timetuple()))) append(_buffer, '}') elif _type is Date: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON) append(_buffer, float2json(value.unix)) append(_buffer, '}') elif _type is timedelta: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON) append(_buffer, float2json(value.total_seconds())) append(_buffer, '}') elif _type is Duration: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON) append(_buffer, float2json(value.seconds)) append(_buffer, '}') elif _type is NullType: append(_buffer, 'null') elif hasattr(value, '__iter__'): if NESTED_TYPE not in sub_schema: sub_schema[NESTED_TYPE] = {} net_new_properties.append(path + [NESTED_TYPE]) append(_buffer, '{' + QUOTED_NESTED_TYPE + COLON) self._iter2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, _buffer) append(_buffer, '}') else: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def typed_encode(value, sub_schema, path, net_new_properties, buffer): """ :param value: THE DATA STRUCTURE TO ENCODE :param sub_schema: dict FROM PATH TO Column DESCRIBING THE TYPE :param path: list OF CURRENT PATH :param net_new_properties: list FOR ADDING NEW PROPERTIES NOT FOUND IN sub_schema :param buffer: UnicodeBuilder OBJECT :return: """ try: # from jx_base import Column if sub_schema.__class__.__name__ == "Column": value_json_type = python_type_to_json_type[value.__class__] column_json_type = es_type_to_json_type[sub_schema.es_type] if value_json_type == column_json_type: pass # ok elif value_json_type == ARRAY and all( python_type_to_json_type[v.__class__] == column_json_type for v in value if v != None): pass # empty arrays can be anything else: from mo_logs import Log Log.error( "Can not store {{value}} in {{column|quote}}", value=value, column=sub_schema.name, ) sub_schema = { json_type_to_inserter_type[value_json_type]: sub_schema } if value == None and path: from mo_logs import Log Log.error("can not encode null (missing) values") elif value is True: if BOOLEAN_KEY not in sub_schema: sub_schema[BOOLEAN_KEY] = {} net_new_properties.append(path + [BOOLEAN_KEY]) append(buffer, "{") append(buffer, QUOTED_BOOLEAN_KEY) append(buffer, "true}") return elif value is False: if BOOLEAN_KEY not in sub_schema: sub_schema[BOOLEAN_KEY] = {} net_new_properties.append(path + [BOOLEAN_KEY]) append(buffer, "{") append(buffer, QUOTED_BOOLEAN_KEY) append(buffer, "false}") return _type = value.__class__ if _type in (dict, Data): if sub_schema.__class__.__name__ == "Column": from mo_logs import Log Log.error("Can not handle {{column|json}}", column=sub_schema) if ARRAY_KEY in sub_schema: # PREFER NESTED, WHEN SEEN BEFORE if value: append(buffer, "{") append(buffer, QUOTED_ARRAY_KEY) append(buffer, "[") _dict2json( value, sub_schema[ARRAY_KEY], path + [ARRAY_KEY], net_new_properties, buffer, ) append(buffer, "]" + COMMA) append(buffer, QUOTED_EXISTS_KEY) append(buffer, text(len(value))) append(buffer, "}") else: # SINGLETON LIST append(buffer, "{") append(buffer, QUOTED_ARRAY_KEY) append(buffer, "[{") append(buffer, QUOTED_EXISTS_KEY) append(buffer, "1}]") append(buffer, COMMA) append(buffer, QUOTED_EXISTS_KEY) append(buffer, "1}") else: if EXISTS_KEY not in sub_schema: sub_schema[EXISTS_KEY] = {} net_new_properties.append(path + [EXISTS_KEY]) if value: _dict2json(value, sub_schema, path, net_new_properties, buffer) else: append(buffer, "{") append(buffer, QUOTED_EXISTS_KEY) append(buffer, "1}") elif _type is binary_type: if STRING_KEY not in sub_schema: sub_schema[STRING_KEY] = True net_new_properties.append(path + [STRING_KEY]) append(buffer, "{") append(buffer, QUOTED_STRING_KEY) append(buffer, '"') try: v = value.decode("utf8") except Exception as e: raise problem_serializing(value, e) for c in v: append(buffer, ESCAPE_DCT.get(c, c)) append(buffer, '"}') elif _type is text: if STRING_KEY not in sub_schema: sub_schema[STRING_KEY] = True net_new_properties.append(path + [STRING_KEY]) append(buffer, "{") append(buffer, QUOTED_STRING_KEY) append(buffer, '"') for c in value: append(buffer, ESCAPE_DCT.get(c, c)) append(buffer, '"}') elif _type in integer_types: if NUMBER_KEY not in sub_schema: sub_schema[NUMBER_KEY] = True net_new_properties.append(path + [NUMBER_KEY]) append(buffer, "{") append(buffer, QUOTED_NUMBER_KEY) append(buffer, text(value)) append(buffer, "}") elif _type in (float, Decimal): if NUMBER_KEY not in sub_schema: sub_schema[NUMBER_KEY] = True net_new_properties.append(path + [NUMBER_KEY]) append(buffer, "{") append(buffer, QUOTED_NUMBER_KEY) append(buffer, float2json(value)) append(buffer, "}") elif _type in (set, list, tuple, FlatList): if len(value) == 0: append(buffer, "{") append(buffer, QUOTED_EXISTS_KEY) append(buffer, "0}") elif any(v.__class__ in (Data, dict, set, list, tuple, FlatList) for v in value): if len(value) == 1: if ARRAY_KEY in sub_schema: append(buffer, "{") append(buffer, QUOTED_ARRAY_KEY) _list2json( value, sub_schema[ARRAY_KEY], path + [ARRAY_KEY], net_new_properties, buffer, ) append(buffer, "}") else: # NO NEED TO NEST, SO DO NOT DO IT typed_encode(value[0], sub_schema, path, net_new_properties, buffer) else: if ARRAY_KEY not in sub_schema: sub_schema[ARRAY_KEY] = {} net_new_properties.append(path + [ARRAY_KEY]) append(buffer, "{") append(buffer, QUOTED_ARRAY_KEY) _list2json( value, sub_schema[ARRAY_KEY], path + [ARRAY_KEY], net_new_properties, buffer, ) append(buffer, "}") else: # ALLOW PRIMITIVE MULTIVALUES value = [v for v in value if v != None] types = list( set(python_type_to_json_type_key[v.__class__] for v in value)) if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM append(buffer, "{") append(buffer, QUOTED_ARRAY_KEY) append(buffer, "[]}") elif len(types) > 1: _list2json( value, sub_schema, path + [ARRAY_KEY], net_new_properties, buffer, ) else: element_type = types[0] if element_type not in sub_schema: sub_schema[element_type] = True net_new_properties.append(path + [element_type]) append(buffer, "{") append(buffer, quote(element_type)) append(buffer, COLON) _multivalue2json( value, sub_schema[element_type], path + [element_type], net_new_properties, buffer, ) append(buffer, "}") elif _type is date: if NUMBER_KEY not in sub_schema: sub_schema[NUMBER_KEY] = True net_new_properties.append(path + [NUMBER_KEY]) append(buffer, "{") append(buffer, QUOTED_NUMBER_KEY) append(buffer, float2json(datetime2unix(value))) append(buffer, "}") elif _type is datetime: if NUMBER_KEY not in sub_schema: sub_schema[NUMBER_KEY] = True net_new_properties.append(path + [NUMBER_KEY]) append(buffer, "{") append(buffer, QUOTED_NUMBER_KEY) append(buffer, float2json(datetime2unix(value))) append(buffer, "}") elif _type is Date: if NUMBER_KEY not in sub_schema: sub_schema[NUMBER_KEY] = True net_new_properties.append(path + [NUMBER_KEY]) append(buffer, "{") append(buffer, QUOTED_NUMBER_KEY) append(buffer, float2json(value.unix)) append(buffer, "}") elif _type is timedelta: if NUMBER_KEY not in sub_schema: sub_schema[NUMBER_KEY] = True net_new_properties.append(path + [NUMBER_KEY]) append(buffer, "{") append(buffer, QUOTED_NUMBER_KEY) append(buffer, float2json(value.total_seconds())) append(buffer, "}") elif _type is Duration: if NUMBER_KEY not in sub_schema: sub_schema[NUMBER_KEY] = True net_new_properties.append(path + [NUMBER_KEY]) append(buffer, "{") append(buffer, QUOTED_NUMBER_KEY) append(buffer, float2json(value.seconds)) append(buffer, "}") elif _type is NullType: append(buffer, "null") elif hasattr(value, "__data__"): typed_encode(value.__data__(), sub_schema, path, net_new_properties, buffer) elif hasattr(value, "__iter__"): if ARRAY_KEY not in sub_schema: sub_schema[ARRAY_KEY] = {} net_new_properties.append(path + [ARRAY_KEY]) append(buffer, "{") append(buffer, QUOTED_ARRAY_KEY) _iter2json( value, sub_schema[ARRAY_KEY], path + [ARRAY_KEY], net_new_properties, buffer, ) append(buffer, "}") else: from mo_logs import Log Log.error(text(repr(value)) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(text(repr(value)) + " is not JSON serializable", cause=e)
def _typed_encode(value, _buffer): try: if value is None: append(_buffer, u'{"$value": null}') return elif value is True: append(_buffer, u'{"$value": true}') return elif value is False: append(_buffer, u'{"$value": false}') return _type = value.__class__ if _type in (dict, Data): if value: _dict2json(value, _buffer) else: append(_buffer, u'{"$object": "."}') elif _type is str: append(_buffer, u'{"$value": "') try: v = utf82unicode(value) except Exception as e: raise problem_serializing(value, e) for c in v: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, u'"}') elif _type is unicode: append(_buffer, u'{"$value": "') for c in value: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, u'"}') elif _type in (int, long, Decimal): append(_buffer, u'{"$value": ') append(_buffer, float2json(value)) append(_buffer, u'}') elif _type is float: append(_buffer, u'{"$value": ') append(_buffer, float2json(value)) append(_buffer, u'}') elif _type in (set, list, tuple, FlatList): _list2json(value, _buffer) elif _type is date: append(_buffer, u'{"$value": ') append(_buffer, float2json(time.mktime(value.timetuple()))) append(_buffer, u'}') elif _type is datetime: append(_buffer, u'{"$value": ') append(_buffer, float2json(time.mktime(value.timetuple()))) append(_buffer, u'}') elif _type is Date: append(_buffer, u'{"$value": ') append(_buffer, float2json(time.mktime(value.value.timetuple()))) append(_buffer, u'}') elif _type is timedelta: append(_buffer, u'{"$value": ') append(_buffer, float2json(value.total_seconds())) append(_buffer, u'}') elif _type is Duration: append(_buffer, u'{"$value": ') append(_buffer, float2json(value.seconds)) append(_buffer, u'}') elif _type is NullType: append(_buffer, u"null") elif hasattr(value, '__json__'): j = value.__json__() t = json2typed(j) append(_buffer, t) elif hasattr(value, '__iter__'): _iter2json(value, _buffer) else: from mo_logs import Log Log.error(_repr(value) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(_repr(value) + " is not JSON serializable", e)
def _typed_encode(value, _buffer): try: if value is None: append(_buffer, u'{"$value": null}') return elif value is True: append(_buffer, u'{"$value": true}') return elif value is False: append(_buffer, u'{"$value": false}') return _type = value.__class__ if _type in (dict, Data): if value: _dict2json(value, _buffer) else: append(_buffer, u'{"$object": "."}') elif _type is str: append(_buffer, u'{"$value": "') try: v = utf82unicode(value) except Exception as e: raise problem_serializing(value, e) for c in v: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, u'"}') elif _type is unicode: append(_buffer, u'{"$value": "') for c in value: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, u'"}') elif _type in (int, long, Decimal): append(_buffer, u'{"$value": ') append(_buffer, float2json(value)) append(_buffer, u'}') elif _type is float: append(_buffer, u'{"$value": ') append(_buffer, float2json(value)) append(_buffer, u'}') elif _type in (set, list, tuple, FlatList): _list2json(value, _buffer) elif _type is date: append(_buffer, u'{"$value": ') append(_buffer, float2json(time.mktime(value.timetuple()))) append(_buffer, u'}') elif _type is datetime: append(_buffer, u'{"$value": ') append(_buffer, float2json(time.mktime(value.timetuple()))) append(_buffer, u'}') elif _type is Date: append(_buffer, u'{"$value": ') append(_buffer, float2json(time.mktime(value.value.timetuple()))) append(_buffer, u'}') elif _type is timedelta: append(_buffer, u'{"$value": ') append(_buffer, float2json(value.total_seconds())) append(_buffer, u'}') elif _type is Duration: append(_buffer, u'{"$value": ') append(_buffer, float2json(value.seconds)) append(_buffer, u'}') elif _type is NullType: append(_buffer, u"null") elif hasattr(value, '__json__'): j = value.__json__() t = json2typed(j) append(_buffer, t) elif hasattr(value, '__iter__'): _iter2json(value, _buffer) else: from mo_logs import Log Log.error(_repr(value) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(_repr(value) + " is not JSON serializable", e)
def typed_encode(value, sub_schema, path, net_new_properties, buffer): """ :param value: THE DATASCRUTURE TO ENCODE :param sub_schema: dict FROM PATH TO Column DESCRIBING THE TYPE :param path: list OF CURRENT PATH :param net_new_properties: list FOR ADDING NEW PROPERTIES NOT FOUND IN sub_schema :param buffer: UnicodeBuilder OBJECT :return: """ try: if isinstance(sub_schema, Column): value_json_type = python_type_to_json_type[value.__class__] column_json_type = es_type_to_json_type[sub_schema.es_type] if value_json_type == column_json_type: pass # ok elif value_json_type == NESTED and all( python_type_to_json_type[v.__class__] == column_json_type for v in value if v != None): pass # empty arrays can be anything else: from mo_logs import Log Log.error("Can not store {{value}} in {{column|quote}}", value=value, column=sub_schema.names['.']) sub_schema = { json_type_to_inserter_type[value_json_type]: sub_schema } if value == None: from mo_logs import Log Log.error("can not encode null (missing) values") elif value is True: if BOOLEAN_TYPE not in sub_schema: sub_schema[BOOLEAN_TYPE] = {} net_new_properties.append(path + [BOOLEAN_TYPE]) append(buffer, '{') append(buffer, QUOTED_BOOLEAN_TYPE) append(buffer, 'true}') return elif value is False: if BOOLEAN_TYPE not in sub_schema: sub_schema[BOOLEAN_TYPE] = {} net_new_properties.append(path + [BOOLEAN_TYPE]) append(buffer, '{') append(buffer, QUOTED_BOOLEAN_TYPE) append(buffer, 'false}') return _type = value.__class__ if _type in (dict, Data): if isinstance(sub_schema, Column): from mo_logs import Log Log.error("Can not handle {{column|json}}", column=sub_schema) if NESTED_TYPE in sub_schema: # PREFER NESTED, WHEN SEEN BEFORE if value: append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) append(buffer, '[') _dict2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, ']' + COMMA) append(buffer, QUOTED_EXISTS_TYPE) append(buffer, text_type(len(value))) append(buffer, '}') else: # SINGLETON LISTS OF null SHOULD NOT EXIST from mo_logs import Log Log.error("should not happen") else: if EXISTS_TYPE not in sub_schema: sub_schema[EXISTS_TYPE] = {} net_new_properties.append(path + [EXISTS_TYPE]) if value: _dict2json(value, sub_schema, path, net_new_properties, buffer) else: append(buffer, '{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '0}') elif _type is binary_type: if STRING_TYPE not in sub_schema: sub_schema[STRING_TYPE] = True net_new_properties.append(path + [STRING_TYPE]) append(buffer, '{') append(buffer, QUOTED_STRING_TYPE) append(buffer, '"') try: v = utf82unicode(value) except Exception as e: raise problem_serializing(value, e) for c in v: append(buffer, ESCAPE_DCT.get(c, c)) append(buffer, '"}') elif _type is text_type: if STRING_TYPE not in sub_schema: sub_schema[STRING_TYPE] = True net_new_properties.append(path + [STRING_TYPE]) append(buffer, '{') append(buffer, QUOTED_STRING_TYPE) append(buffer, '"') for c in value: append(buffer, ESCAPE_DCT.get(c, c)) append(buffer, '"}') elif _type in (int, long, Decimal): if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value)) append(buffer, '}') elif _type is float: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value)) append(buffer, '}') elif _type in (set, list, tuple, FlatList): if len(value) == 0: append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) append(buffer, '[]}') elif any( isinstance(v, (Mapping, set, list, tuple, FlatList)) for v in value): if NESTED_TYPE not in sub_schema: sub_schema[NESTED_TYPE] = {} net_new_properties.append(path + [NESTED_TYPE]) append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) _list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, '}') else: # ALLOW PRIMITIVE MULTIVALUES value = [v for v in value if v != None] types = list( set(json_type_to_inserter_type[python_type_to_json_type[ v.__class__]] for v in value)) if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) append(buffer, '[]}') elif len(types) > 1: _list2json(value, sub_schema, path + [NESTED_TYPE], net_new_properties, buffer) else: element_type = types[0] if element_type not in sub_schema: sub_schema[element_type] = True net_new_properties.append(path + [element_type]) append(buffer, '{') append(buffer, quote(element_type)) append(buffer, COLON) _multivalue2json(value, sub_schema[element_type], path + [element_type], net_new_properties, buffer) append(buffer, '}') elif _type is date: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(time.mktime(value.timetuple()))) append(buffer, '}') elif _type is datetime: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(time.mktime(value.timetuple()))) append(buffer, '}') elif _type is Date: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value.unix)) append(buffer, '}') elif _type is timedelta: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value.total_seconds())) append(buffer, '}') elif _type is Duration: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value.seconds)) append(buffer, '}') elif _type is NullType: append(buffer, 'null') elif hasattr(value, '__data__'): typed_encode(value.__data__(), sub_schema, path, net_new_properties, buffer) elif hasattr(value, '__iter__'): if NESTED_TYPE not in sub_schema: sub_schema[NESTED_TYPE] = {} net_new_properties.append(path + [NESTED_TYPE]) append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) _iter2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, '}') else: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)