def typed_encode(self, r):
"""
:param record: expecting id and value properties
:return: dict with id and json properties
"""
try:
value = r['value']
if "json" in r:
value = json2value(r["json"])
elif isinstance(value, Mapping) or value != None:
pass
else:
from mo_logs import Log
raise Log.error(
"Expecting every record given to have \"value\" or \"json\" property"
)
_buffer = UnicodeBuilder(1024)
net_new_properties = []
path = []
if isinstance(value, Mapping):
given_id = self.get_id(value)
if self.remove_id:
value['_id'] = None
else:
given_id = None
if given_id:
record_id = r.get('id')
if record_id and record_id != given_id:
from mo_logs import Log
raise Log.error(
"expecting {{property}} of record ({{record_id|quote}}) to match one given ({{given|quote}})",
property=self.id_column,
record_id=record_id,
given=given_id)
else:
record_id = r.get('id')
if record_id:
given_id = record_id
else:
given_id = random_id()
typed_encode(value, self.schema, path, net_new_properties, _buffer)
json = _buffer.build()
for props in net_new_properties:
path, type = props[:-1], props[-1][1:]
# self.es.add_column(join_field(path), type)
return {"id": given_id, "json": json}
except Exception as e:
# THE PRETTY JSON WILL PROVIDE MORE DETAIL ABOUT THE SERIALIZATION CONCERNS
from mo_logs import Log
Log.error("Serialization of JSON problems", cause=e)
def typed_encode(self, record):
"""
:param record: expecting id and value properties
:return: dict with id and json properties
"""
try:
value = record.get('value')
if "json" in record:
value = json2value(record["json"])
elif is_data(value) or value != None:
pass
else:
from mo_logs import Log
raise Log.error(
"Expecting every record given to have \"value\" or \"json\" property"
)
_buffer = UnicodeBuilder(1024)
net_new_properties = []
path = []
if is_data(value):
given_id = self.get_id(value)
if given_id != None and not isinstance(given_id, text):
given_id = value2json(given_id)
value['_id'] = None
version = self.get_version(value)
else:
given_id = None
version = None
if given_id:
record_id = record.get('id')
if record_id and record_id != given_id:
from mo_logs import Log
raise Log.error(
"expecting {{property}} of record ({{record_id|quote}}) to match one given ({{given|quote}})",
property=self.id_info,
record_id=record_id,
given=given_id)
else:
record_id = record.get('id')
if record_id:
given_id = record_id
else:
given_id = random_id()
typed_encode(value, self.schema, path, net_new_properties, _buffer)
json = _buffer.build()
return given_id, version, json
except Exception as e:
# THE PRETTY JSON WILL PROVIDE MORE DETAIL ABOUT THE SERIALIZATION CONCERNS
from mo_logs import Log
Log.error("Serialization of JSON problems", cause=e)
def test_double(self):
value = {"value": 5.2025595183536973e-07}
test1 = typed_encode(value)
expected = u'{"value":{' + quote(
NUMBER_KEY) + u':5.202559518353697e-7},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_list_of_objects(self):
value = {"a": [{}, "b"]}
test1 = typed_encode(value)
expected = u'{"a":{"' + ARRAY_KEY + u'":[{' + quote(
EXISTS_KEY) + u':1},{' + quote(STRING_KEY) + u':"b"}]},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_null_object_in_list(self):
value = wrap({"a": [{"b": None}]})
test = typed_encode(value)
expected = u'{"a":{' + quote(EXISTS_KEY) + u':1},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test, expected)
def test_unicode2(self):
value = {
"comment": "testing accented char àáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
}
test1 = typed_encode(value)
expected = u'{"comment":{' + quote(
STRING_KEY
) + u':"testing accented char àáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_unicode3(self):
value = {
"comment":
u"testing accented char ŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙"
}
test1 = typed_encode(value)
expected = u'{"comment":{' + quote(
STRING_KEY
) + u':"testing accented char ŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙"},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_unicode1(self):
value = {
"comment":
u"Open all links in the current tab, except the pages opened from external apps — open these ones in new windows"
}
test1 = typed_encode(value)
expected = u'{"comment":{' + quote(
STRING_KEY
) + u':"Open all links in the current tab, except the pages opened from external apps — open these ones in new windows"},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_number_value(self):
value = 42
test1 = typed_encode(value)
expected = '{' + quote(NUMBER_KEY) + u':42}'
self.assertEqual(test1, expected)
self.assertEqual(expected, test1)
def test_list_value(self):
value = [42]
test1 = typed_encode(value)
expected = u'{' + quote(INTEGER_KEY) + u':42}'
self.assertEqual(test1, expected)
def test_empty_list1(self):
value = wrap({"a": []})
test1 = typed_encode(value)
expected = u'{' + quote(EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_complex_object(self):
value = wrap({"s": 0, "r": 5})
test1 = typed_encode(value)
expected = u'{"r":{' + quote(NUMBER_KEY) + u':5},"s":{' + quote(
NUMBER_KEY) + u':0},' + quote(EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_empty_dict(self):
value = wrap({"match_all": wrap({})})
test1 = typed_encode(value)
expected = u'{"match_all":{' + quote(EXISTS_KEY) + u':1},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def encode_null():
value = None
typed_encode(value)
def test_true(self):
value = True
test1 = typed_encode(value)
expected = '{' + quote(BOOLEAN_KEY) + u':true}'
self.assertEqual(test1, expected)
def test_false(self):
value = False
test1 = typed_encode(value)
expected = '{' + quote(BOOLEAN_KEY) + u':false}'
self.assertEqual(test1, expected)
def test_escaped_string_value(self):
value = "\""
test1 = typed_encode(value)
expected = '{' + quote(STRING_KEY) + u':"\\""}'
self.assertEqual(test1, expected)
def test_empty_list(self):
value = {"value": []}
test1 = typed_encode(value)
expected = u'{"value":{' + quote(EXISTS_KEY) + u':0},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_nested(self):
value = {"a": {}, "b": {}}
test1 = typed_encode(value)
expected = u'{"a":{' + quote(EXISTS_KEY) + u':1},"b":{' + quote(
EXISTS_KEY) + u':1},' + quote(EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_empty_list2(self):
value = wrap({"a": [], "b": 1})
test1 = typed_encode(value)
expected = u'{"b":{' + quote(NUMBER_KEY) + u':1},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_empty_list_value(self):
value = []
test1 = typed_encode(value)
expected = u'{' + quote(EXISTS_KEY) + ':0}'
self.assertEqual(test1, expected)
def test_list_i(self):
value = {"value": [23, 42]}
test1 = typed_encode(value)
expected = u'{"value":{' + quote(INTEGER_KEY) + u':[23,42]},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_date(self):
value = {"test": datetime.date(2013, 11, 13)}
test1 = typed_encode(value)
expected = u'{"test":{' + quote(NUMBER_KEY) + u':1384300800},' + quote(
EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
def test_list_n(self):
value = {"value": [23.5, 42]}
test1 = typed_encode(value)
expected = u'{"value":[{' + quote(NUMBER_KEY) + u':23.5},{' + quote(
NUMBER_KEY) + u':42}],' + quote(EXISTS_KEY) + u':1}'
self.assertEqual(test1, expected)
