def encode(value):
buffer = UnicodeBuilder(1024)
typed_encode(value,
sub_schema={},
path=[],
net_new_properties=[],
buffer=buffer)
return buffer.build()
def typed_encode(self, r):
"""
:param record: expecting id and value properties
:return: dict with id and json properties
"""
try:
value = r['value']
if "json" in r:
value = json2value(r["json"])
elif isinstance(value, Mapping) or value != None:
pass
else:
from mo_logs import Log
raise Log.error(
"Expecting every record given to have \"value\" or \"json\" property"
)
_buffer = UnicodeBuilder(1024)
net_new_properties = []
path = []
if isinstance(value, Mapping):
given_id = self.get_id(value)
if self.remove_id:
value['_id'] = None
else:
given_id = None
if given_id:
record_id = r.get('id')
if record_id and record_id != given_id:
from mo_logs import Log
raise Log.error(
"expecting {{property}} of record ({{record_id|quote}}) to match one given ({{given|quote}})",
property=self.id_column,
record_id=record_id,
given=given_id)
else:
record_id = r.get('id')
if record_id:
given_id = record_id
else:
given_id = random_id()
self._typed_encode(value, self.schema, path, net_new_properties,
_buffer)
json = _buffer.build()
for props in net_new_properties:
path, type = props[:-1], props[-1][1:]
# self.es.add_column(join_field(path), type)
return {"id": given_id, "json": json}
except Exception as e:
# THE PRETTY JSON WILL PROVIDE MORE DETAIL ABOUT THE SERIALIZATION CONCERNS
from mo_logs import Log
Log.error("Serialization of JSON problems", cause=e)
def typed_encode(self, record):
"""
:param record: expecting id and value properties
:return: dict with id and json properties
"""
try:
value = record.get('value')
if "json" in record:
value = json2value(record["json"])
elif is_data(value) or value != None:
pass
else:
from mo_logs import Log
raise Log.error(
"Expecting every record given to have \"value\" or \"json\" property"
)
_buffer = UnicodeBuilder(1024)
net_new_properties = []
path = []
if is_data(value):
given_id = self.get_id(value)
if given_id != None and not isinstance(given_id, text):
given_id = value2json(given_id)
value['_id'] = None
version = self.get_version(value)
else:
given_id = None
version = None
if given_id:
record_id = record.get('id')
if record_id and record_id != given_id:
from mo_logs import Log
raise Log.error(
"expecting {{property}} of record ({{record_id|quote}}) to match one given ({{given|quote}})",
property=self.id_info,
record_id=record_id,
given=given_id)
else:
record_id = record.get('id')
if record_id:
given_id = record_id
else:
given_id = random_id()
typed_encode(value, self.schema, path, net_new_properties, _buffer)
json = _buffer.build()
return given_id, version, json
except Exception as e:
# THE PRETTY JSON WILL PROVIDE MORE DETAIL ABOUT THE SERIALIZATION CONCERNS
from mo_logs import Log
Log.error("Serialization of JSON problems", cause=e)
def encode(value):
buffer = UnicodeBuilder(1024)
typed_encode(
value,
sub_schema={},
path=[],
net_new_properties=[],
buffer=buffer
)
return buffer.build()
def typed_encode(value):
"""
pypy DOES NOT OPTIMIZE GENERATOR CODE WELL
"""
try:
_buffer = UnicodeBuilder(1024)
_typed_encode(value, _buffer)
output = _buffer.build()
return output
except Exception as e:
# THE PRETTY JSON WILL PROVIDE MORE DETAIL ABOUT THE SERIALIZATION CONCERNS
from mo_logs import Log
Log.warning("Serialization of JSON problems", e)
try:
return pretty_json(value)
except Exception, f:
Log.error("problem serializing object", f)
def typed_encode(value):
"""
pypy DOES NOT OPTIMIZE GENERATOR CODE WELL
"""
try:
_buffer = UnicodeBuilder(1024)
_typed_encode(value, _buffer)
output = _buffer.build()
return output
except Exception as e:
# THE PRETTY JSON WILL PROVIDE MORE DETAIL ABOUT THE SERIALIZATION CONCERNS
from mo_logs import Log
Log.warning("Serialization of JSON problems", e)
try:
return pretty_json(value)
except Exception, f:
Log.error("problem serializing object", f)
def parse_string(i, json):
j = i
output = UnicodeBuilder()
while True:
c = json[j]
if c == "\"":
return j + 1, output.build()
elif c == "\\":
j += 1
c = json[j]
if c == "u":
n = json[j:j + 4].decode('hex').decode('utf-8')
output.append(n)
j += 4
else:
try:
output.append(ESC[c])
except Exception as e:
output.append("\\")
output.append(c)
else:
output.append(c)
j += 1
def json2typed(json):
"""
every ': {' gets converted to ': {"$object": ".", '
every ': <value>' gets converted to '{"$value": <value>}'
"""
# MODE VALUES
#
context = deque()
output = UnicodeBuilder(1024)
mode = VALUE
for c in json:
if c in "\t\r\n ":
append(output, c)
elif mode == VALUE:
if c == "{":
context.append(mode)
mode = BEGIN_OBJECT
append(output, '{"$object": "."')
continue
elif c == '[':
context.append(mode)
mode = VALUE
elif c == ",":
mode = context.pop()
if mode != OBJECT:
context.append(mode)
mode = VALUE
elif c in "]":
mode = context.pop()
elif c in "}":
mode = context.pop()
mode = context.pop()
elif c == '"':
context.append(mode)
mode = STRING
append(output, '{"$value": ')
else:
mode = PRIMITIVE
append(output, '{"$value": ')
append(output, c)
elif mode == PRIMITIVE:
if c == ",":
append(output, '}')
mode = context.pop()
if mode == 0:
context.append(mode)
elif c == "]":
append(output, '}')
mode = context.pop()
elif c == "}":
append(output, '}')
mode = context.pop()
mode = context.pop()
append(output, c)
elif mode == BEGIN_OBJECT:
if c == '"':
context.append(OBJECT)
context.append(KEYWORD)
mode = STRING
append(output, ', ')
elif c == "}":
mode = context.pop()
else:
Log.error("not expected")
append(output, c)
elif mode == KEYWORD:
append(output, c)
if c == ':':
mode = VALUE
else:
Log.error("Not expected")
elif mode == STRING:
append(output, c)
if c == '"':
mode = context.pop()
if mode != KEYWORD:
append(output, '}')
elif c == '\\':
context.append(mode)
mode = ESCAPE
elif mode == ESCAPE:
mode = context.pop()
append(output, c)
elif mode == OBJECT:
if c == '"':
context.append(mode)
context.append(KEYWORD)
mode = STRING
elif c == ",":
pass
elif c == '}':
mode = context.pop()
else:
Log.error("not expected")
append(output, c)
if mode == PRIMITIVE:
append(output, "}")
return output.build()
def json2typed(json):
"""
every ': {' gets converted to ': {"$object": ".", '
every ': <value>' gets converted to '{"$value": <value>}'
"""
# MODE VALUES
#
context = deque()
output = UnicodeBuilder(1024)
mode = VALUE
for c in json:
if c in "\t\r\n ":
append(output, c)
elif mode == VALUE:
if c == "{":
context.append(mode)
mode = BEGIN_OBJECT
append(output, '{"$object": "."')
continue
elif c == '[':
context.append(mode)
mode = VALUE
elif c == ",":
mode = context.pop()
if mode != OBJECT:
context.append(mode)
mode = VALUE
elif c in "]":
mode = context.pop()
elif c in "}":
mode = context.pop()
mode = context.pop()
elif c == '"':
context.append(mode)
mode = STRING
append(output, '{"$value": ')
else:
mode = PRIMITIVE
append(output, '{"$value": ')
append(output, c)
elif mode == PRIMITIVE:
if c == ",":
append(output, '}')
mode = context.pop()
if mode == 0:
context.append(mode)
elif c == "]":
append(output, '}')
mode = context.pop()
elif c == "}":
append(output, '}')
mode = context.pop()
mode = context.pop()
append(output, c)
elif mode == BEGIN_OBJECT:
if c == '"':
context.append(OBJECT)
context.append(KEYWORD)
mode = STRING
append(output, ', ')
elif c == "}":
mode = context.pop()
else:
Log.error("not expected")
append(output, c)
elif mode == KEYWORD:
append(output, c)
if c == ':':
mode = VALUE
else:
Log.error("Not expected")
elif mode == STRING:
append(output, c)
if c == '"':
mode = context.pop()
if mode != KEYWORD:
append(output, '}')
elif c == '\\':
context.append(mode)
mode = ESCAPE
elif mode == ESCAPE:
mode = context.pop()
append(output, c)
elif mode == OBJECT:
if c == '"':
context.append(mode)
context.append(KEYWORD)
mode = STRING
elif c == ",":
pass
elif c == '}':
mode = context.pop()
else:
Log.error("not expected")
append(output, c)
if mode == PRIMITIVE:
append(output, "}")
return output.build()
def parse_string(i, json):
j = i
output = UnicodeBuilder()
while True:
c = json[j]
if c == "\"":
return j + 1, output.build()
elif c == "\\":
j += 1
c = json[j]
if c == "u":
n = json[j:j + 4].decode('hex').decode('utf-8')
output.append(n)
j += 4
else:
try:
output.append(ESC[c])
except Exception as e:
output.append("\\")
output.append(c)
else:
output.append(c)
j += 1