def write(self, template, params):
output = {
"Timestamp": (Decimal(datetime2unix(params.timestamp)) *
Decimal(1e9)).to_integral_exact(), # NANOSECONDS
"Type":
params.template,
"Logger":
params.machine.name,
"Hostname":
self.app_name,
"EnvVersion":
"2.0",
"Severity":
severity_map.get(
params.context,
3), # https://en.wikipedia.org/wiki/Syslog#Severity_levels
"Pid":
params.machine.pid,
"Fields": {
k: _deep_json_to_string(v, 0)
for k, v in to_data(params).leaves()
}
}
self.stream.write(value2json(output).encode('utf8'))
self.stream.write(b'\n')
def _deep_json_to_string(value, depth):
"""
:param value: SOME STRUCTURE
:param depth: THE MAX DEPTH OF PROPERTIES, DEEPER WILL BE STRING-IFIED
:return: FLATTER STRUCTURE
"""
if is_data(value):
if depth == 0:
return strings.limit(value2json(value), LOG_STRING_LENGTH)
return {
k: _deep_json_to_string(v, depth - 1)
for k, v in value.items()
}
elif is_sequence(value):
return strings.limit(value2json(value), LOG_STRING_LENGTH)
elif isinstance(value, number_types):
return value
elif is_text(value):
return strings.limit(value, LOG_STRING_LENGTH)
elif is_binary(value):
return strings.limit(bytes2base64(value), LOG_STRING_LENGTH)
elif isinstance(value, (date, datetime)):
return datetime2unix(value)
else:
return strings.limit(value2json(value), LOG_STRING_LENGTH)
def write(self, template, params):
output = {
"Timestamp": (Decimal(datetime2unix(params.timestamp)) * Decimal(1e9)).to_integral_exact(), # NANOSECONDS
"Type": params.template,
"Logger": params.machine.name,
"Hostname": self.app_name,
"EnvVersion": "2.0",
"Severity": severity_map.get(params.context, 3), # https://en.wikipedia.org/wiki/Syslog#Severity_levels
"Pid": params.machine.pid,
"Fields": {
k: _deep_json_to_string(v, 0)
for k, v in wrap(params).leaves()
}
}
self.stream.write(value2json(output).encode('utf8'))
self.stream.write(b'\n')
def _deep_json_to_string(value, depth):
"""
:param value: SOME STRUCTURE
:param depth: THE MAX DEPTH OF PROPERTIES, DEEPER WILL BE STRING-IFIED
:return: FLATTER STRUCTURE
"""
if is_data(value):
if depth == 0:
return strings.limit(value2json(value), LOG_STRING_LENGTH)
return {k: _deep_json_to_string(v, depth - 1) for k, v in value.items()}
elif is_sequence(value):
return strings.limit(value2json(value), LOG_STRING_LENGTH)
elif isinstance(value, number_types):
return value
elif is_text(value):
return strings.limit(value, LOG_STRING_LENGTH)
elif is_binary(value):
return strings.limit(bytes2base64(value), LOG_STRING_LENGTH)
elif isinstance(value, (date, datetime)):
return datetime2unix(value)
else:
return strings.limit(value2json(value), LOG_STRING_LENGTH)
def typed_encode(value, sub_schema, path, net_new_properties, buffer):
"""
:param value: THE DATA STRUCTURE TO ENCODE
:param sub_schema: dict FROM PATH TO Column DESCRIBING THE TYPE
:param path: list OF CURRENT PATH
:param net_new_properties: list FOR ADDING NEW PROPERTIES NOT FOUND IN sub_schema
:param buffer: UnicodeBuilder OBJECT
:return:
"""
try:
# from jx_base import Column
if sub_schema.__class__.__name__ == "Column":
value_json_type = python_type_to_json_type[value.__class__]
column_json_type = es_type_to_json_type[sub_schema.es_type]
if value_json_type == column_json_type:
pass # ok
elif value_json_type == ARRAY and all(
python_type_to_json_type[v.__class__] == column_json_type
for v in value if v != None):
pass # empty arrays can be anything
else:
from mo_logs import Log
Log.error(
"Can not store {{value}} in {{column|quote}}",
value=value,
column=sub_schema.name,
)
sub_schema = {
json_type_to_inserter_type[value_json_type]: sub_schema
}
if value == None and path:
from mo_logs import Log
Log.error("can not encode null (missing) values")
elif value is True:
if BOOLEAN_KEY not in sub_schema:
sub_schema[BOOLEAN_KEY] = {}
net_new_properties.append(path + [BOOLEAN_KEY])
append(buffer, "{")
append(buffer, QUOTED_BOOLEAN_KEY)
append(buffer, "true}")
return
elif value is False:
if BOOLEAN_KEY not in sub_schema:
sub_schema[BOOLEAN_KEY] = {}
net_new_properties.append(path + [BOOLEAN_KEY])
append(buffer, "{")
append(buffer, QUOTED_BOOLEAN_KEY)
append(buffer, "false}")
return
_type = value.__class__
if _type in (dict, Data):
if sub_schema.__class__.__name__ == "Column":
from mo_logs import Log
Log.error("Can not handle {{column|json}}", column=sub_schema)
if ARRAY_KEY in sub_schema:
# PREFER NESTED, WHEN SEEN BEFORE
if value:
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
append(buffer, "[")
_dict2json(
value,
sub_schema[ARRAY_KEY],
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
append(buffer, "]" + COMMA)
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, text(len(value)))
append(buffer, "}")
else:
# SINGLETON LIST
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
append(buffer, "[{")
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "1}]")
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "1}")
else:
if EXISTS_KEY not in sub_schema:
sub_schema[EXISTS_KEY] = {}
net_new_properties.append(path + [EXISTS_KEY])
if value:
_dict2json(value, sub_schema, path, net_new_properties,
buffer)
else:
append(buffer, "{")
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "1}")
elif _type is binary_type:
if STRING_KEY not in sub_schema:
sub_schema[STRING_KEY] = True
net_new_properties.append(path + [STRING_KEY])
append(buffer, "{")
append(buffer, QUOTED_STRING_KEY)
append(buffer, '"')
try:
v = value.decode("utf8")
except Exception as e:
raise problem_serializing(value, e)
for c in v:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type is text:
if STRING_KEY not in sub_schema:
sub_schema[STRING_KEY] = True
net_new_properties.append(path + [STRING_KEY])
append(buffer, "{")
append(buffer, QUOTED_STRING_KEY)
append(buffer, '"')
for c in value:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type in integer_types:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, text(value))
append(buffer, "}")
elif _type in (float, Decimal):
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(value))
append(buffer, "}")
elif _type in (set, list, tuple, FlatList):
if len(value) == 0:
append(buffer, "{")
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "0}")
elif any(v.__class__ in (Data, dict, set, list, tuple, FlatList)
for v in value):
if len(value) == 1:
if ARRAY_KEY in sub_schema:
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
_list2json(
value,
sub_schema[ARRAY_KEY],
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
append(buffer, "}")
else:
# NO NEED TO NEST, SO DO NOT DO IT
typed_encode(value[0], sub_schema, path,
net_new_properties, buffer)
else:
if ARRAY_KEY not in sub_schema:
sub_schema[ARRAY_KEY] = {}
net_new_properties.append(path + [ARRAY_KEY])
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
_list2json(
value,
sub_schema[ARRAY_KEY],
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
append(buffer, "}")
else:
# ALLOW PRIMITIVE MULTIVALUES
value = [v for v in value if v != None]
types = list(
set(python_type_to_json_type_key[v.__class__]
for v in value))
if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
append(buffer, "[]}")
elif len(types) > 1:
_list2json(
value,
sub_schema,
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
else:
element_type = types[0]
if element_type not in sub_schema:
sub_schema[element_type] = True
net_new_properties.append(path + [element_type])
append(buffer, "{")
append(buffer, quote(element_type))
append(buffer, COLON)
_multivalue2json(
value,
sub_schema[element_type],
path + [element_type],
net_new_properties,
buffer,
)
append(buffer, "}")
elif _type is date:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(datetime2unix(value)))
append(buffer, "}")
elif _type is datetime:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(datetime2unix(value)))
append(buffer, "}")
elif _type is Date:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(value.unix))
append(buffer, "}")
elif _type is timedelta:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(value.total_seconds()))
append(buffer, "}")
elif _type is Duration:
if NUMBER_KEY not in sub_schema:
sub_schema[NUMBER_KEY] = True
net_new_properties.append(path + [NUMBER_KEY])
append(buffer, "{")
append(buffer, QUOTED_NUMBER_KEY)
append(buffer, float2json(value.seconds))
append(buffer, "}")
elif _type is NullType:
append(buffer, "null")
elif hasattr(value, "__data__"):
typed_encode(value.__data__(), sub_schema, path,
net_new_properties, buffer)
elif hasattr(value, "__iter__"):
if ARRAY_KEY not in sub_schema:
sub_schema[ARRAY_KEY] = {}
net_new_properties.append(path + [ARRAY_KEY])
append(buffer, "{")
append(buffer, QUOTED_ARRAY_KEY)
_iter2json(
value,
sub_schema[ARRAY_KEY],
path + [ARRAY_KEY],
net_new_properties,
buffer,
)
append(buffer, "}")
else:
from mo_logs import Log
Log.error(text(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text(repr(value)) + " is not JSON serializable", cause=e)