def _dict2json(value, _buffer):
prefix = u'{"$object": ".", "'
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
if not isinstance(k, unicode):
Log.error("Expecting property name to be a string")
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_typed_encode(v, _buffer)
append(_buffer, u"}")
def _dict2json(value, _buffer):
prefix = u'{"$object": ".", "'
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
if not isinstance(k, unicode):
Log.error("Expecting property name to be a string")
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_typed_encode(v, _buffer)
append(_buffer, u"}")
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.items():
append(_buffer, prefix)
prefix = COMMA_QUOTE
if is_binary(k):
k = k.decode('utf8')
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE_COLON)
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception as e:
from mo_logs import Log
Log.error(text(repr(value)) + " is not JSON serializable", cause=e)
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.items():
append(_buffer, prefix)
prefix = COMMA_QUOTE
if is_binary(k):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE_COLON)
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception as e:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable", cause=e)
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception as e:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable", cause=e)
def _value2json(value, _buffer):
try:
_class = value.__class__
if value is None:
append(_buffer, u"null")
return
elif value is True:
append(_buffer, u"true")
return
elif value is False:
append(_buffer, u"false")
return
type = value.__class__
if type is binary_type:
append(_buffer, QUOTE)
try:
v = utf82unicode(value)
except Exception as e:
problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE)
elif type is text_type:
append(_buffer, QUOTE)
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE)
elif type is dict:
if not value:
append(_buffer, u"{}")
else:
_dict2json(value, _buffer)
return
elif type is Data:
d = _get(value, "_dict") # MIGHT BE A VALUE NOT A DICT
_value2json(d, _buffer)
return
elif type in (int, long, Decimal):
append(_buffer, text_type(value))
elif type is float:
if math.isnan(value) or math.isinf(value):
append(_buffer, u'null')
else:
append(_buffer, float2json(value))
elif type in (set, list, tuple, FlatList):
_list2json(value, _buffer)
elif type is date:
append(_buffer, float2json(time.mktime(value.timetuple())))
elif type is datetime:
append(_buffer, float2json(time.mktime(value.timetuple())))
elif type is Date:
append(_buffer, float2json(value.unix))
elif type is timedelta:
append(_buffer, float2json(value.total_seconds()))
elif type is Duration:
append(_buffer, float2json(value.seconds))
elif type is NullType:
append(_buffer, u"null")
elif isinstance(value, Mapping):
if not value:
append(_buffer, u"{}")
else:
_dict2json(value, _buffer)
return
elif hasattr(value, '__data__'):
d = value.__data__()
_value2json(d, _buffer)
elif hasattr(value, '__json__'):
j = value.__json__()
append(_buffer, j)
elif hasattr(value, '__iter__'):
_iter2json(value, _buffer)
else:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable",
cause=e)
def _value2json(value, _buffer):
try:
_class = value.__class__
if value is None:
append(_buffer, u"null")
return
elif value is True:
append(_buffer, u"true")
return
elif value is False:
append(_buffer, u"false")
return
type = value.__class__
if type is binary_type:
append(_buffer, QUOTE)
try:
v = utf82unicode(value)
except Exception as e:
problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE)
elif type is text_type:
append(_buffer, QUOTE)
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE)
elif type is dict:
if not value:
append(_buffer, u"{}")
else:
_dict2json(value, _buffer)
return
elif type is Data:
d = _get(value, SLOT) # MIGHT BE A VALUE NOT A DICT
_value2json(d, _buffer)
return
elif type in (int, long, Decimal):
append(_buffer, text_type(value))
elif type is float:
if math.isnan(value) or math.isinf(value):
append(_buffer, u'null')
else:
append(_buffer, float2json(value))
elif type in (set, list, tuple, FlatList):
_list2json(value, _buffer)
elif type is date:
append(_buffer, float2json(time.mktime(value.timetuple())))
elif type is datetime:
append(_buffer, float2json(time.mktime(value.timetuple())))
elif type is Date:
append(_buffer, float2json(value.unix))
elif type is timedelta:
append(_buffer, float2json(value.total_seconds()))
elif type is Duration:
append(_buffer, float2json(value.seconds))
elif type is NullType:
append(_buffer, u"null")
elif is_data(value):
if not value:
append(_buffer, u"{}")
else:
_dict2json(value, _buffer)
return
elif hasattr(value, '__data__'):
d = value.__data__()
_value2json(d, _buffer)
elif hasattr(value, '__json__'):
j = value.__json__()
append(_buffer, j)
elif hasattr(value, '__iter__'):
_iter2json(value, _buffer)
else:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def typed_encode(value, sub_schema, path, net_new_properties, buffer):
"""
:param value: THE DATA STRUCTURE TO ENCODE
:param sub_schema: dict FROM PATH TO Column DESCRIBING THE TYPE
:param path: list OF CURRENT PATH
:param net_new_properties: list FOR ADDING NEW PROPERTIES NOT FOUND IN sub_schema
:param buffer: UnicodeBuilder OBJECT
:return:
"""
try:
# from jx_base import Column
if sub_schema.__class__.__name__=='Column':
value_json_type = python_type_to_json_type[value.__class__]
column_json_type = es_type_to_json_type[sub_schema.es_type]
if value_json_type == column_json_type:
pass # ok
elif value_json_type == NESTED and all(python_type_to_json_type[v.__class__] == column_json_type for v in value if v != None):
pass # empty arrays can be anything
else:
from mo_logs import Log
Log.error("Can not store {{value}} in {{column|quote}}", value=value, column=sub_schema.name)
sub_schema = {json_type_to_inserter_type[value_json_type]: sub_schema}
if value == None:
from mo_logs import Log
Log.error("can not encode null (missing) values")
elif value is True:
if BOOLEAN_TYPE not in sub_schema:
sub_schema[BOOLEAN_TYPE] = {}
net_new_properties.append(path + [BOOLEAN_TYPE])
append(buffer, '{')
append(buffer, QUOTED_BOOLEAN_TYPE)
append(buffer, 'true}')
return
elif value is False:
if BOOLEAN_TYPE not in sub_schema:
sub_schema[BOOLEAN_TYPE] = {}
net_new_properties.append(path + [BOOLEAN_TYPE])
append(buffer, '{')
append(buffer, QUOTED_BOOLEAN_TYPE)
append(buffer, 'false}')
return
_type = value.__class__
if _type in (dict, Data):
if sub_schema.__class__.__name__ == 'Column':
from mo_logs import Log
Log.error("Can not handle {{column|json}}", column=sub_schema)
if NESTED_TYPE in sub_schema:
# PREFER NESTED, WHEN SEEN BEFORE
if value:
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
append(buffer, '[')
_dict2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer)
append(buffer, ']' + COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, text_type(len(value)))
append(buffer, '}')
else:
# SINGLETON LIST
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
append(buffer, '[{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}]')
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
else:
if EXISTS_TYPE not in sub_schema:
sub_schema[EXISTS_TYPE] = {}
net_new_properties.append(path + [EXISTS_TYPE])
if value:
_dict2json(value, sub_schema, path, net_new_properties, buffer)
else:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
elif _type is binary_type:
if STRING_TYPE not in sub_schema:
sub_schema[STRING_TYPE] = True
net_new_properties.append(path + [STRING_TYPE])
append(buffer, '{')
append(buffer, QUOTED_STRING_TYPE)
append(buffer, '"')
try:
v = utf82unicode(value)
except Exception as e:
raise problem_serializing(value, e)
for c in v:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type is text_type:
if STRING_TYPE not in sub_schema:
sub_schema[STRING_TYPE] = True
net_new_properties.append(path + [STRING_TYPE])
append(buffer, '{')
append(buffer, QUOTED_STRING_TYPE)
append(buffer, '"')
for c in value:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type in (int, long):
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, text_type(value))
append(buffer, '}')
elif _type in (float, Decimal):
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value))
append(buffer, '}')
elif _type in (set, list, tuple, FlatList):
if len(value) == 0:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '0}')
elif any(v.__class__ in (Data, dict, set, list, tuple, FlatList) for v in value):
# THIS IS NOT DONE BECAUSE
if len(value) == 1:
if NESTED_TYPE in sub_schema:
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
_list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer)
append(buffer, '}')
else:
# NO NEED TO NEST, SO DO NOT DO IT
typed_encode(value[0], sub_schema, path, net_new_properties, buffer)
else:
if NESTED_TYPE not in sub_schema:
sub_schema[NESTED_TYPE] = {}
net_new_properties.append(path + [NESTED_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
_list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer)
append(buffer, '}')
else:
# ALLOW PRIMITIVE MULTIVALUES
value = [v for v in value if v != None]
types = list(set(json_type_to_inserter_type[python_type_to_json_type[v.__class__]] for v in value))
if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
append(buffer, '[]}')
elif len(types) > 1:
_list2json(value, sub_schema, path + [NESTED_TYPE], net_new_properties, buffer)
else:
element_type = types[0]
if element_type not in sub_schema:
sub_schema[element_type] = True
net_new_properties.append(path + [element_type])
append(buffer, '{')
append(buffer, quote(element_type))
append(buffer, COLON)
_multivalue2json(value, sub_schema[element_type], path + [element_type], net_new_properties, buffer)
append(buffer, '}')
elif _type is date:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(time.mktime(value.timetuple())))
append(buffer, '}')
elif _type is datetime:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(time.mktime(value.timetuple())))
append(buffer, '}')
elif _type is Date:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value.unix))
append(buffer, '}')
elif _type is timedelta:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value.total_seconds()))
append(buffer, '}')
elif _type is Duration:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value.seconds))
append(buffer, '}')
elif _type is NullType:
append(buffer, 'null')
elif hasattr(value, '__data__'):
typed_encode(value.__data__(), sub_schema, path, net_new_properties, buffer)
elif hasattr(value, '__iter__'):
if NESTED_TYPE not in sub_schema:
sub_schema[NESTED_TYPE] = {}
net_new_properties.append(path + [NESTED_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
_iter2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer)
append(buffer, '}')
else:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def _typed_encode(self, value, sub_schema, path, net_new_properties,
_buffer):
try:
if isinstance(sub_schema, Column):
value_json_type = python_type_to_json_type[value.__class__]
column_json_type = es_type_to_json_type[sub_schema.es_type]
if value_json_type == column_json_type:
pass # ok
elif value_json_type == NESTED and all(
python_type_to_json_type[
v.__class__] == column_json_type
for v in value if v != None):
pass # empty arrays can be anything
else:
from mo_logs import Log
Log.error("Can not store {{value}} in {{column|quote}}",
value=value,
column=sub_schema.names['.'])
sub_schema = {
json_type_to_inserter_type[value_json_type]: sub_schema
}
if value is None:
append(_buffer, '{}')
return
elif value is True:
if BOOLEAN_TYPE not in sub_schema:
sub_schema[BOOLEAN_TYPE] = {}
net_new_properties.append(path + [BOOLEAN_TYPE])
append(_buffer, '{' + QUOTED_BOOLEAN_TYPE + COLON + 'true}')
return
elif value is False:
if BOOLEAN_TYPE not in sub_schema:
sub_schema[BOOLEAN_TYPE] = {}
net_new_properties.append(path + [BOOLEAN_TYPE])
append(_buffer, '{' + QUOTED_BOOLEAN_TYPE + COLON + 'false}')
return
_type = value.__class__
if _type in (dict, Data):
if isinstance(sub_schema, Column):
from mo_logs import Log
Log.error("Can not handle {{column|json}}",
column=sub_schema)
if NESTED_TYPE in sub_schema:
# PREFER NESTED, WHEN SEEN BEFORE
if value:
append(_buffer, '{' + QUOTED_NESTED_TYPE + COLON + '[')
self._dict2json(value, sub_schema[NESTED_TYPE],
path + [NESTED_TYPE],
net_new_properties, _buffer)
append(
_buffer, ']' + COMMA + QUOTED_EXISTS_TYPE + COLON +
text_type(len(value)) + '}')
else:
# SINGLETON LISTS OF null SHOULD NOT EXIST
from mo_logs import Log
Log.error("should not happen")
else:
if EXISTS_TYPE not in sub_schema:
sub_schema[EXISTS_TYPE] = {}
net_new_properties.append(path + [EXISTS_TYPE])
if value:
self._dict2json(value, sub_schema, path,
net_new_properties, _buffer)
else:
append(_buffer,
'{' + QUOTED_EXISTS_TYPE + COLON + '0}')
elif _type is binary_type:
if STRING_TYPE not in sub_schema:
sub_schema[STRING_TYPE] = True
net_new_properties.append(path + [STRING_TYPE])
append(_buffer, '{' + QUOTED_STRING_TYPE + COLON + '"')
try:
v = utf82unicode(value)
except Exception as e:
raise problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, '"}')
elif _type is text_type:
if STRING_TYPE not in sub_schema:
sub_schema[STRING_TYPE] = True
net_new_properties.append(path + [STRING_TYPE])
append(_buffer, '{' + QUOTED_STRING_TYPE + COLON + '"')
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, '"}')
elif _type in (int, long, Decimal):
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON)
append(_buffer, float2json(value))
append(_buffer, '}')
elif _type is float:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON)
append(_buffer, float2json(value))
append(_buffer, '}')
elif _type in (set, list, tuple, FlatList):
if len(value) == 0:
append(_buffer, '{' + QUOTED_NESTED_TYPE + COLON + '[]}')
elif any(
isinstance(v, (Mapping, set, list, tuple, FlatList))
for v in value):
if NESTED_TYPE not in sub_schema:
sub_schema[NESTED_TYPE] = {}
net_new_properties.append(path + [NESTED_TYPE])
append(_buffer, '{' + QUOTED_NESTED_TYPE + COLON)
self._list2json(value, sub_schema[NESTED_TYPE],
path + [NESTED_TYPE], net_new_properties,
_buffer)
append(_buffer, '}')
else:
# ALLOW PRIMITIVE MULTIVALUES
value = [v for v in value if v != None]
types = list(
set(python_type_to_json_type[v.__class__]
for v in value))
if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM
append(_buffer,
'{' + QUOTED_NESTED_TYPE + COLON + '[]}')
elif len(types) > 1:
from mo_logs import Log
Log.error("Can not handle multi-typed multivalues")
else:
element_type = json_type_to_inserter_type[types[0]]
if element_type not in sub_schema:
sub_schema[element_type] = True
net_new_properties.append(path + [element_type])
append(_buffer, '{' + quote(element_type) + COLON)
self._multivalue2json(value, sub_schema[element_type],
path + [element_type],
net_new_properties, _buffer)
append(_buffer, '}')
elif _type is date:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON)
append(_buffer, float2json(time.mktime(value.timetuple())))
append(_buffer, '}')
elif _type is datetime:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON)
append(_buffer, float2json(time.mktime(value.timetuple())))
append(_buffer, '}')
elif _type is Date:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON)
append(_buffer, float2json(value.unix))
append(_buffer, '}')
elif _type is timedelta:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON)
append(_buffer, float2json(value.total_seconds()))
append(_buffer, '}')
elif _type is Duration:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(_buffer, '{' + QUOTED_NUMBER_TYPE + COLON)
append(_buffer, float2json(value.seconds))
append(_buffer, '}')
elif _type is NullType:
append(_buffer, 'null')
elif hasattr(value, '__iter__'):
if NESTED_TYPE not in sub_schema:
sub_schema[NESTED_TYPE] = {}
net_new_properties.append(path + [NESTED_TYPE])
append(_buffer, '{' + QUOTED_NESTED_TYPE + COLON)
self._iter2json(value, sub_schema[NESTED_TYPE],
path + [NESTED_TYPE], net_new_properties,
_buffer)
append(_buffer, '}')
else:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable",
cause=e)
def typed_encode(value, sub_schema, path, net_new_properties, buffer):
"""
:param value: THE DATA STRUCTURE TO ENCODE
:param sub_schema: dict FROM PATH TO Column DESCRIBING THE TYPE
:param path: list OF CURRENT PATH
:param net_new_properties: list FOR ADDING NEW PROPERTIES NOT FOUND IN sub_schema
:param buffer: UnicodeBuilder OBJECT
:return:
"""
try:
# from jx_base import Column
if sub_schema.__class__.__name__ == "Column":
value_json_type = python_type_to_json_type[value.__class__]
column_json_type = es_type_to_json_type[sub_schema.es_type]
if value_json_type == column_json_type:
pass # ok
elif value_json_type == ARRAY and all(
python_type_to_json_type[v.__class__] == column_json_type
for v in value if v != None):
pass # empty arrays can be anything
else:
from mo_logs import Log
Log.error(
"Can not store {{value}} in {{column|quote}}",
value=value,
column=sub_schema.name,
)
sub_schema = {
json_type_to_inserter_type[value_json_type]: sub_schema
}
if value == None and path:
from mo_logs import Log
Log.error("can not encode null (missing) values")
elif value is True:
if BOOLEAN_KEY not in sub_schema:
sub_schema[BOOLEAN_KEY] = {}
net_new_properties.append(path + [BOOLEAN_KEY])
append(buffer, "{")
append(buffer, QUOTED_BOOLEAN_KEY)
append(buffer, "true}")
return
elif value is False:
if BOOLEAN_KEY not in sub_schema:
sub_schema[BOOLEAN_KEY] = {}
net_new_properties.append(path + [BOOLEAN_KEY])
append(buffer, "{")
append(buffer, QUOTED_BOOLEAN_KEY)
append(buffer, "false}")
return
_type = value.__class__
if _type in (dict, Data):
if sub_schema.__class__.__name__ == "Column":
from mo_logs import Log
Log.error("Can not handle {{column|json}}", column=sub_schema)
if ARRAY_KEY in sub_schema:
# PREFER NESTED, WHEN SEEN BEFORE
if value:
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
append(buffer, "[")
_dict2json(
value,
sub_schema[ARRAY_KEY],
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
append(buffer, "]" + COMMA)
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, text(len(value)))
append(buffer, "}")
else:
# SINGLETON LIST
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
append(buffer, "[{")
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "1}]")
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "1}")
else:
if EXISTS_KEY not in sub_schema:
sub_schema[EXISTS_KEY] = {}
net_new_properties.append(path + [EXISTS_KEY])
if value:
_dict2json(value, sub_schema, path, net_new_properties,
buffer)
else:
append(buffer, "{")
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "1}")
elif _type is binary_type:
if STRING_KEY not in sub_schema:
sub_schema[STRING_KEY] = True
net_new_properties.append(path + [STRING_KEY])
append(buffer, "{")
append(buffer, QUOTED_STRING_KEY)
append(buffer, '"')
try:
v = value.decode("utf8")
except Exception as e:
raise problem_serializing(value, e)
for c in v:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type is text:
if STRING_KEY not in sub_schema:
sub_schema[STRING_KEY] = True
net_new_properties.append(path + [STRING_KEY])
append(buffer, "{")
append(buffer, QUOTED_STRING_KEY)
append(buffer, '"')
for c in value:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type in integer_types:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, text(value))
append(buffer, "}")
elif _type in (float, Decimal):
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(value))
append(buffer, "}")
elif _type in (set, list, tuple, FlatList):
if len(value) == 0:
append(buffer, "{")
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "0}")
elif any(v.__class__ in (Data, dict, set, list, tuple, FlatList)
for v in value):
if len(value) == 1:
if ARRAY_KEY in sub_schema:
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
_list2json(
value,
sub_schema[ARRAY_KEY],
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
append(buffer, "}")
else:
# NO NEED TO NEST, SO DO NOT DO IT
typed_encode(value[0], sub_schema, path,
net_new_properties, buffer)
else:
if ARRAY_KEY not in sub_schema:
sub_schema[ARRAY_KEY] = {}
net_new_properties.append(path + [ARRAY_KEY])
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
_list2json(
value,
sub_schema[ARRAY_KEY],
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
append(buffer, "}")
else:
# ALLOW PRIMITIVE MULTIVALUES
value = [v for v in value if v != None]
types = list(
set(python_type_to_json_type_key[v.__class__]
for v in value))
if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
append(buffer, "[]}")
elif len(types) > 1:
_list2json(
value,
sub_schema,
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
else:
element_type = types[0]
if element_type not in sub_schema:
sub_schema[element_type] = True
net_new_properties.append(path + [element_type])
append(buffer, "{")
append(buffer, quote(element_type))
append(buffer, COLON)
_multivalue2json(
value,
sub_schema[element_type],
path + [element_type],
net_new_properties,
buffer,
)
append(buffer, "}")
elif _type is date:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(datetime2unix(value)))
append(buffer, "}")
elif _type is datetime:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(datetime2unix(value)))
append(buffer, "}")
elif _type is Date:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(value.unix))
append(buffer, "}")
elif _type is timedelta:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(value.total_seconds()))
append(buffer, "}")
elif _type is Duration:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(value.seconds))
append(buffer, "}")
elif _type is NullType:
append(buffer, "null")
elif hasattr(value, "__data__"):
typed_encode(value.__data__(), sub_schema, path,
net_new_properties, buffer)
elif hasattr(value, "__iter__"):
if ARRAY_KEY not in sub_schema:
sub_schema[ARRAY_KEY] = {}
net_new_properties.append(path + [ARRAY_KEY])
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
_iter2json(
value,
sub_schema[ARRAY_KEY],
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
append(buffer, "}")
else:
from mo_logs import Log
Log.error(text(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text(repr(value)) + " is not JSON serializable", cause=e)
def _typed_encode(value, _buffer):
try:
if value is None:
append(_buffer, u'{"$value": null}')
return
elif value is True:
append(_buffer, u'{"$value": true}')
return
elif value is False:
append(_buffer, u'{"$value": false}')
return
_type = value.__class__
if _type in (dict, Data):
if value:
_dict2json(value, _buffer)
else:
append(_buffer, u'{"$object": "."}')
elif _type is str:
append(_buffer, u'{"$value": "')
try:
v = utf82unicode(value)
except Exception as e:
raise problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u'"}')
elif _type is unicode:
append(_buffer, u'{"$value": "')
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u'"}')
elif _type in (int, long, Decimal):
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value))
append(_buffer, u'}')
elif _type is float:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value))
append(_buffer, u'}')
elif _type in (set, list, tuple, FlatList):
_list2json(value, _buffer)
elif _type is date:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(time.mktime(value.timetuple())))
append(_buffer, u'}')
elif _type is datetime:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(time.mktime(value.timetuple())))
append(_buffer, u'}')
elif _type is Date:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(time.mktime(value.value.timetuple())))
append(_buffer, u'}')
elif _type is timedelta:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value.total_seconds()))
append(_buffer, u'}')
elif _type is Duration:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value.seconds))
append(_buffer, u'}')
elif _type is NullType:
append(_buffer, u"null")
elif hasattr(value, '__json__'):
j = value.__json__()
t = json2typed(j)
append(_buffer, t)
elif hasattr(value, '__iter__'):
_iter2json(value, _buffer)
else:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable", e)
def _typed_encode(value, _buffer):
try:
if value is None:
append(_buffer, u'{"$value": null}')
return
elif value is True:
append(_buffer, u'{"$value": true}')
return
elif value is False:
append(_buffer, u'{"$value": false}')
return
_type = value.__class__
if _type in (dict, Data):
if value:
_dict2json(value, _buffer)
else:
append(_buffer, u'{"$object": "."}')
elif _type is str:
append(_buffer, u'{"$value": "')
try:
v = utf82unicode(value)
except Exception as e:
raise problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u'"}')
elif _type is unicode:
append(_buffer, u'{"$value": "')
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u'"}')
elif _type in (int, long, Decimal):
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value))
append(_buffer, u'}')
elif _type is float:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value))
append(_buffer, u'}')
elif _type in (set, list, tuple, FlatList):
_list2json(value, _buffer)
elif _type is date:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(time.mktime(value.timetuple())))
append(_buffer, u'}')
elif _type is datetime:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(time.mktime(value.timetuple())))
append(_buffer, u'}')
elif _type is Date:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(time.mktime(value.value.timetuple())))
append(_buffer, u'}')
elif _type is timedelta:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value.total_seconds()))
append(_buffer, u'}')
elif _type is Duration:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value.seconds))
append(_buffer, u'}')
elif _type is NullType:
append(_buffer, u"null")
elif hasattr(value, '__json__'):
j = value.__json__()
t = json2typed(j)
append(_buffer, t)
elif hasattr(value, '__iter__'):
_iter2json(value, _buffer)
else:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable", e)
def typed_encode(value, sub_schema, path, net_new_properties, buffer):
"""
:param value: THE DATASCRUTURE TO ENCODE
:param sub_schema: dict FROM PATH TO Column DESCRIBING THE TYPE
:param path: list OF CURRENT PATH
:param net_new_properties: list FOR ADDING NEW PROPERTIES NOT FOUND IN sub_schema
:param buffer: UnicodeBuilder OBJECT
:return:
"""
try:
if isinstance(sub_schema, Column):
value_json_type = python_type_to_json_type[value.__class__]
column_json_type = es_type_to_json_type[sub_schema.es_type]
if value_json_type == column_json_type:
pass # ok
elif value_json_type == NESTED and all(
python_type_to_json_type[v.__class__] == column_json_type
for v in value if v != None):
pass # empty arrays can be anything
else:
from mo_logs import Log
Log.error("Can not store {{value}} in {{column|quote}}",
value=value,
column=sub_schema.names['.'])
sub_schema = {
json_type_to_inserter_type[value_json_type]: sub_schema
}
if value == None:
from mo_logs import Log
Log.error("can not encode null (missing) values")
elif value is True:
if BOOLEAN_TYPE not in sub_schema:
sub_schema[BOOLEAN_TYPE] = {}
net_new_properties.append(path + [BOOLEAN_TYPE])
append(buffer, '{')
append(buffer, QUOTED_BOOLEAN_TYPE)
append(buffer, 'true}')
return
elif value is False:
if BOOLEAN_TYPE not in sub_schema:
sub_schema[BOOLEAN_TYPE] = {}
net_new_properties.append(path + [BOOLEAN_TYPE])
append(buffer, '{')
append(buffer, QUOTED_BOOLEAN_TYPE)
append(buffer, 'false}')
return
_type = value.__class__
if _type in (dict, Data):
if isinstance(sub_schema, Column):
from mo_logs import Log
Log.error("Can not handle {{column|json}}", column=sub_schema)
if NESTED_TYPE in sub_schema:
# PREFER NESTED, WHEN SEEN BEFORE
if value:
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
append(buffer, '[')
_dict2json(value, sub_schema[NESTED_TYPE],
path + [NESTED_TYPE], net_new_properties,
buffer)
append(buffer, ']' + COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, text_type(len(value)))
append(buffer, '}')
else:
# SINGLETON LISTS OF null SHOULD NOT EXIST
from mo_logs import Log
Log.error("should not happen")
else:
if EXISTS_TYPE not in sub_schema:
sub_schema[EXISTS_TYPE] = {}
net_new_properties.append(path + [EXISTS_TYPE])
if value:
_dict2json(value, sub_schema, path, net_new_properties,
buffer)
else:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '0}')
elif _type is binary_type:
if STRING_TYPE not in sub_schema:
sub_schema[STRING_TYPE] = True
net_new_properties.append(path + [STRING_TYPE])
append(buffer, '{')
append(buffer, QUOTED_STRING_TYPE)
append(buffer, '"')
try:
v = utf82unicode(value)
except Exception as e:
raise problem_serializing(value, e)
for c in v:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type is text_type:
if STRING_TYPE not in sub_schema:
sub_schema[STRING_TYPE] = True
net_new_properties.append(path + [STRING_TYPE])
append(buffer, '{')
append(buffer, QUOTED_STRING_TYPE)
append(buffer, '"')
for c in value:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type in (int, long, Decimal):
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value))
append(buffer, '}')
elif _type is float:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value))
append(buffer, '}')
elif _type in (set, list, tuple, FlatList):
if len(value) == 0:
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
append(buffer, '[]}')
elif any(
isinstance(v, (Mapping, set, list, tuple, FlatList))
for v in value):
if NESTED_TYPE not in sub_schema:
sub_schema[NESTED_TYPE] = {}
net_new_properties.append(path + [NESTED_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
_list2json(value, sub_schema[NESTED_TYPE],
path + [NESTED_TYPE], net_new_properties, buffer)
append(buffer, '}')
else:
# ALLOW PRIMITIVE MULTIVALUES
value = [v for v in value if v != None]
types = list(
set(json_type_to_inserter_type[python_type_to_json_type[
v.__class__]] for v in value))
if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
append(buffer, '[]}')
elif len(types) > 1:
_list2json(value, sub_schema, path + [NESTED_TYPE],
net_new_properties, buffer)
else:
element_type = types[0]
if element_type not in sub_schema:
sub_schema[element_type] = True
net_new_properties.append(path + [element_type])
append(buffer, '{')
append(buffer, quote(element_type))
append(buffer, COLON)
_multivalue2json(value, sub_schema[element_type],
path + [element_type], net_new_properties,
buffer)
append(buffer, '}')
elif _type is date:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(time.mktime(value.timetuple())))
append(buffer, '}')
elif _type is datetime:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(time.mktime(value.timetuple())))
append(buffer, '}')
elif _type is Date:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value.unix))
append(buffer, '}')
elif _type is timedelta:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value.total_seconds()))
append(buffer, '}')
elif _type is Duration:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value.seconds))
append(buffer, '}')
elif _type is NullType:
append(buffer, 'null')
elif hasattr(value, '__data__'):
typed_encode(value.__data__(), sub_schema, path,
net_new_properties, buffer)
elif hasattr(value, '__iter__'):
if NESTED_TYPE not in sub_schema:
sub_schema[NESTED_TYPE] = {}
net_new_properties.append(path + [NESTED_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
_iter2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE],
net_new_properties, buffer)
append(buffer, '}')
else:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable",
cause=e)