def write(self, key, value, disable_zip=False):
if key.endswith(".json") or key.endswith(".zip"):
Log.error("Expecting a pure key")
try:
if hasattr(value, "read"):
if disable_zip:
storage = self.bucket.new_key(str(key + ".json"))
string_length = len(value)
headers = {"Content-Type": mimetype.JSON}
else:
storage = self.bucket.new_key(str(key + ".json.gz"))
string_length = len(value)
value = convert.bytes2zip(value)
headers = {"Content-Type": mimetype.GZIP}
file_length = len(value)
Log.note(
"Sending contents with length {{file_length|comma}} (from string with length {{string_length|comma}})",
file_length=file_length,
string_length=string_length,
)
value.seek(0)
storage.set_contents_from_file(value, headers=headers)
if self.settings.public:
storage.set_acl("public-read")
return
if len(value) > 20 * 1000 and not disable_zip:
self.bucket.delete_key(str(key + ".json"))
self.bucket.delete_key(str(key + ".json.gz"))
if is_binary(value):
value = convert.bytes2zip(value)
key += ".json.gz"
else:
value = convert.bytes2zip(value).encode("utf8")
key += ".json.gz"
headers = {"Content-Type": mimetype.GZIP}
else:
self.bucket.delete_key(str(key + ".json.gz"))
if is_binary(value):
key += ".json"
else:
key += ".json"
headers = {"Content-Type": mimetype.JSON}
storage = self.bucket.new_key(str(key))
storage.set_contents_from_string(value, headers=headers)
if self.settings.public:
storage.set_acl("public-read")
except Exception as e:
Log.error(
"Problem writing {{bytes}} bytes to {{key}} in {{bucket}}",
key=key,
bucket=self.bucket.name,
bytes=len(value),
cause=e,
)
def write(self, key, value, disable_zip=False):
if key.endswith(".json") or key.endswith(".zip"):
Log.error("Expecting a pure key")
try:
if hasattr(value, "read"):
if disable_zip:
storage = self.bucket.new_key(key + ".json")
string_length = len(value)
else:
storage = self.bucket.new_key(key + ".json.gz")
string_length = len(value)
value = convert.bytes2zip(value)
file_length = len(value)
Log.note(
"Sending contents with length {{file_length|comma}} (from string with length {{string_length|comma}})",
file_length=file_length,
string_length=string_length)
value.seek(0)
storage.set_contents_from_file(value)
if self.settings.public:
storage.set_acl('public-read')
return
if len(value) > 20 * 1000 and not disable_zip:
self.bucket.delete_key(key + ".json")
self.bucket.delete_key(key + ".json.gz")
if is_binary(value):
value = convert.bytes2zip(value)
key += ".json.gz"
else:
value = convert.bytes2zip(unicode2utf8(value))
key += ".json.gz"
else:
self.bucket.delete_key(key + ".json.gz")
if is_binary(value):
key += ".json"
else:
key += ".json"
storage = self.bucket.new_key(key)
storage.set_contents_from_string(value)
if self.settings.public:
storage.set_acl('public-read')
except Exception as e:
Log.error(
"Problem writing {{bytes}} bytes to {{key}} in {{bucket}}",
key=key,
bucket=self.bucket.name,
bytes=len(value),
cause=e)
def _dict2json(value, sub_schema, path, net_new_properties, buffer):
prefix = "{"
for k, v in sort_using_key(value.items(), lambda r: r[0]):
if v == None or v == "":
continue
append(buffer, prefix)
prefix = COMMA
if is_binary(k):
k = k.decode("utf8")
if not is_text(k):
Log.error("Expecting property name to be a string")
if k not in sub_schema:
sub_schema[k] = {}
net_new_properties.append(path + [k])
append(buffer, quote(encode_property(k)))
append(buffer, COLON)
typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer)
if prefix is COMMA:
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "1}")
else:
append(buffer, "{")
append(buffer, QUOTED_EXISTS_KEY)
append(buffer, "1}")
def value2url_param(value):
"""
:param value:
:return: ascii URL
"""
if value == None:
Log.error("Can not encode None into a URL")
if is_data(value):
from mo_json import value2json
value_ = wrap(value)
output = "&".join([
value2url_param(k) + "=" + (value2url_param(v) if is_text(v) else
value2url_param(value2json(v)))
for k, v in value_.leaves()
])
elif is_text(value):
output = "".join(_map2url[c] for c in value.encode('utf8'))
elif is_binary(value):
output = "".join(_map2url[c] for c in value)
elif hasattr(value, "__iter__"):
output = ",".join(value2url_param(v) for v in value)
else:
output = str(value)
return output
def utf82unicode(value):
"""
WITH EXPLANATION FOR FAILURE
"""
try:
return value.decode("utf8")
except Exception as e:
if not _Log:
_late_import()
if not is_binary(value):
_Log.error("Can not convert {{type}} to unicode because it's not bytes", type= type(value).__name__)
e = _Except.wrap(e)
for i, c in enumerate(value):
try:
c.decode("utf8")
except Exception as f:
_Log.error("Can not convert charcode {{c}} in string index {{i}}", i=i, c=ord(c), cause=[e, _Except.wrap(f)])
try:
latin1 = text_type(value.decode("latin1"))
_Log.error("Can not explain conversion failure, but seems to be latin1", e)
except Exception:
pass
try:
a = text_type(value.decode("latin1"))
_Log.error("Can not explain conversion failure, but seems to be latin1", e)
except Exception:
pass
_Log.error("Can not explain conversion failure of " + type(value).__name__ + "!", e)
def _deep_json_to_string(value, depth):
"""
:param value: SOME STRUCTURE
:param depth: THE MAX DEPTH OF PROPERTIES, DEEPER WILL BE STRING-IFIED
:return: FLATTER STRUCTURE
"""
if is_data(value):
if depth == 0:
return strings.limit(value2json(value), LOG_STRING_LENGTH)
return {
k: _deep_json_to_string(v, depth - 1)
for k, v in value.items()
}
elif is_sequence(value):
return strings.limit(value2json(value), LOG_STRING_LENGTH)
elif isinstance(value, number_types):
return value
elif is_text(value):
return strings.limit(value, LOG_STRING_LENGTH)
elif is_binary(value):
return strings.limit(bytes2base64(value), LOG_STRING_LENGTH)
elif isinstance(value, (date, datetime)):
return datetime2unix(value)
else:
return strings.limit(value2json(value), LOG_STRING_LENGTH)
def string2url(value):
if is_text(value):
return "".join([_map2url[c] for c in unicode2latin1(value)])
elif is_binary(value):
return "".join([_map2url[c] for c in value])
else:
Log.error("Expecting a string")
def _dict2json(value, sub_schema, path, net_new_properties, buffer):
prefix = '{'
for k, v in sort_using_key(value.items(), lambda r: r[0]):
if v == None or v == '':
continue
append(buffer, prefix)
prefix = COMMA
if is_binary(k):
k = utf82unicode(k)
if not is_text(k):
Log.error("Expecting property name to be a string")
if k not in sub_schema:
sub_schema[k] = {}
net_new_properties.append(path + [k])
append(buffer, encode_basestring(encode_property(k)))
append(buffer, COLON)
typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer)
if prefix is COMMA:
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
else:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
def _scrub(result):
if is_text(result):
return result
elif is_binary(result):
return result.decode('utf8')
elif isinstance(result, number_types):
return result
elif not result:
return {}
elif isinstance(result, (list, ParseResults)):
if not result:
return None
elif len(result) == 1:
return _scrub(result[0])
else:
output = [rr for r in result for rr in [_scrub(r)] if rr != None]
# IF ALL MEMBERS OF A LIST ARE LITERALS, THEN MAKE THE LIST LITERAL
if all(is_data(r) and "literal" in r.keys() for r in output):
output = {"literal": [r['literal'] for r in output]}
return output
elif not list(result.items()):
return {}
else:
return {
k: vv
for k, v in result.items() for vv in [_scrub(v)] if vv != None
}
def string2url(value):
if is_text(value):
return "".join([_map2url[c] for c in unicode2latin1(value)])
elif is_binary(value):
return "".join([_map2url[c] for c in value])
else:
Log.error("Expecting a string")
def _dict2json(value, sub_schema, path, net_new_properties, buffer):
prefix = '{'
for k, v in sort_using_key(value.items(), lambda r: r[0]):
if v == None or v == '':
continue
append(buffer, prefix)
prefix = COMMA
if is_binary(k):
k = utf82unicode(k)
if not is_text(k):
Log.error("Expecting property name to be a string")
if k not in sub_schema:
sub_schema[k] = {}
net_new_properties.append(path + [k])
append(buffer, encode_basestring(encode_property(k)))
append(buffer, COLON)
typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer)
if prefix is COMMA:
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
else:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
def encrypt(text, _key, salt=None):
"""
RETURN {"salt":s, "length":l, "data":d} -> JSON -> UTF8
"""
if is_text(text):
encoding = "utf8"
data = bytearray(text.encode("utf8"))
elif is_binary(text):
encoding = None
if PY2:
data = bytearray(text)
else:
data = text
if _key is None:
Log.error("Expecting a key")
if is_binary(_key):
_key = bytearray(_key)
if salt is None:
salt = crypto.bytes(16)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
output.encoding = encoding
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output, pretty=True).encode("utf8")
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def encrypt(text, _key, salt=None):
"""
RETURN {"salt":s, "length":l, "data":d} -> JSON -> UTF8
"""
if is_text(text):
encoding = 'utf8'
data = bytearray(text.encode("utf8"))
elif is_binary(text):
encoding = None
if PY2:
data = bytearray(text)
else:
data = text
if _key is None:
Log.error("Expecting a key")
if is_binary(_key):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
output.encoding = encoding
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output, pretty=True).encode('utf8')
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def utf8_to_unicode(v):
try:
if is_binary(v):
return v.decode("utf8")
else:
return v
except Exception as e:
Log.error("not expected", e)
def utf8_to_unicode(v):
try:
if is_binary(v):
return v.decode("utf8")
else:
return v
except Exception as e:
Log.error("not expected", e)
def parse(timestr, parserinfo=None, **kwargs):
# Python 2.x support: datetimes return their string presentation as
# bytes in 2.x and unicode in 3.x, so it's reasonable to expect that
# the parser will get both kinds. Internally we use unicode only.
if is_binary(timestr):
timestr = timestr.decode()
if parserinfo:
return parser(parserinfo).parse(timestr, **kwargs)
else:
return DEFAULTPARSER.parse(timestr, **kwargs)
def __getattr__(self, key):
if is_binary(key):
ukey = key.decode("utf8")
else:
ukey = key
d = self
o = dict.get(d, ukey, None)
if o == None:
return NullType(d, ukey)
return wrap(o)
def __delitem__(self, key):
if is_binary(key):
key = key.decode("utf8")
if key.find(".") == -1:
dict.pop(self, key, None)
return
d = self
seq = _split_field(key)
for k in seq[:-1]:
d = d[k]
d.pop(seq[-1], None)
def __setattr__(self, key, value):
if is_binary(key):
ukey = key.decode("utf8")
else:
ukey = key
d = self
value = unwrap(value)
if value is None:
dict.pop(d, key, None)
else:
dict.__setitem__(d, ukey, value)
return self
def __getitem__(self, key):
if isinstance(key, slice):
return Null
elif is_binary(key):
key = key.decode("utf8")
elif isinstance(key, int):
return NullType(self, key)
path = _split_field(key)
output = self
for p in path:
output = NullType(output, p)
return output
def __getitem__(self, key):
if isinstance(key, slice):
return Null
elif is_binary(key):
key = key.decode("utf8")
elif isinstance(key, int):
return NullType(self, key)
path = _split_field(key)
output = self
for p in path:
output = NullType(output, p)
return output
def _leaves_to_data(value):
"""
RETURN UNWRAPPED STRUCTURES
"""
if value == None:
return None
class_ = _get(value, CLASS)
if class_ in (text, binary_type, int, float):
return value
if class_ in data_types:
if class_ is Data:
value = from_data(value)
output = {}
for key, value in value.items():
value = _leaves_to_data(value)
if key == "":
get_logger().error("key is empty string. Probably a bad idea")
if is_binary(key):
key = key.decode("utf8")
d = output
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
else:
seq = split_field(key)
for k in seq[:-1]:
e = d.get(k, None)
if e is None:
d[k] = {}
e = d[k]
d = e
if value == None:
d.pop(seq[-1], None)
else:
d[seq[-1]] = value
return output
if hasattr(value, "__iter__"):
output = []
for v in value:
v = leaves_to_data(v)
output.append(v)
return output
return value
def _wrap_leaves(value):
if value == None:
return None
class_ = _get(value, CLASS)
if class_ in (text_type, binary_type, int, float):
return value
if class_ in data_types:
if class_ is Data:
value = unwrap(value)
output = {}
for key, value in value.items():
value = _wrap_leaves(value)
if key == "":
get_logger().error("key is empty string. Probably a bad idea")
if is_binary(key):
key = key.decode("utf8")
d = output
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
else:
seq = split_field(key)
for k in seq[:-1]:
e = d.get(k, None)
if e is None:
d[k] = {}
e = d[k]
d = e
if value == None:
d.pop(seq[-1], None)
else:
d[seq[-1]] = value
return output
if hasattr(value, '__iter__'):
output = []
for v in value:
v = wrap_leaves(v)
output.append(v)
return output
return value
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.items():
append(_buffer, prefix)
prefix = COMMA_QUOTE
if is_binary(k):
k = k.decode('utf8')
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE_COLON)
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception as e:
from mo_logs import Log
Log.error(text(repr(value)) + " is not JSON serializable", cause=e)
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.items():
append(_buffer, prefix)
prefix = COMMA_QUOTE
if is_binary(k):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE_COLON)
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def quote_column(column_name, table=None):
if column_name == None:
Log.error("missing column_name")
elif is_text(column_name):
if table:
return join_column(table, column_name)
else:
return SQL("`" + '`.`'.join(split_field(column_name)) + "`") # MYSQL QUOTE OF COLUMN NAMES
elif is_binary(column_name):
return quote_column(column_name.decode('utf8'), table)
elif is_list(column_name):
if table:
return sql_list(join_column(table, c) for c in column_name)
return sql_list(quote_column(c) for c in column_name)
else:
# ASSUME {"name":name, "value":value} FORM
return SQL(sql_alias(column_name.value, quote_column(column_name.name)))
def utf82unicode(value):
"""
WITH EXPLANATION FOR FAILURE
"""
try:
return value.decode("utf8")
except Exception as e:
if not _Log:
_late_import()
if not is_binary(value):
_Log.error(
"Can not convert {{type}} to unicode because it's not bytes",
type=type(value).__name__)
e = _Except.wrap(e)
for i, c in enumerate(value):
try:
c.decode("utf8")
except Exception as f:
_Log.error(
"Can not convert charcode {{c}} in string index {{i}}",
i=i,
c=ord(c),
cause=[e, _Except.wrap(f)])
try:
latin1 = text_type(value.decode("latin1"))
_Log.error(
"Can not explain conversion failure, but seems to be latin1",
e)
except Exception:
pass
try:
a = text_type(value.decode("latin1"))
_Log.error(
"Can not explain conversion failure, but seems to be latin1",
e)
except Exception:
pass
_Log.error(
"Can not explain conversion failure of " + type(value).__name__ +
"!", e)
def __getitem__(self, key):
if key == None:
return Null
if is_binary(key):
key = key.decode("utf8")
d=self
if key.find(".") >= 0:
seq = _split_field(key)
for n in seq:
d = _getdefault(self, n)
return wrap(d)
else:
o = dict.get(d, None)
if o == None:
return NullType(d, key)
return wrap(o)
def quote_column(column_name, table=None):
if column_name == None:
Log.error("missing column_name")
elif is_text(column_name):
if table:
return join_column(table, column_name)
else:
return SQL("`" + '`.`'.join(split_field(column_name)) +
"`") # MYSQL QUOTE OF COLUMN NAMES
elif is_binary(column_name):
return quote_column(column_name.decode('utf8'), table)
elif is_list(column_name):
if table:
return sql_list(join_column(table, c) for c in column_name)
return sql_list(quote_column(c) for c in column_name)
else:
# ASSUME {"name":name, "value":value} FORM
return SQL(sql_alias(column_name.value,
quote_column(column_name.name)))
def value2url_param(value):
"""
:param value:
:return: ascii URL
"""
if value == None:
Log.error("Can not encode None into a URL")
if is_data(value):
value_ = wrap(value)
output = "&".join([
value2url_param(k) + "=" + (value2url_param(v) if is_text(v) else value2url_param(value2json(v)))
for k, v in value_.leaves()
])
elif is_text(value):
output = "".join(_map2url[c] for c in value.encode('utf8'))
elif is_binary(value):
output = "".join(_map2url[c] for c in value)
elif hasattr(value, "__iter__"):
output = ",".join(value2url_param(v) for v in value)
else:
output = str(value)
return output
def _deep_json_to_string(value, depth):
"""
:param value: SOME STRUCTURE
:param depth: THE MAX DEPTH OF PROPERTIES, DEEPER WILL BE STRING-IFIED
:return: FLATTER STRUCTURE
"""
if is_data(value):
if depth == 0:
return strings.limit(value2json(value), LOG_STRING_LENGTH)
return {k: _deep_json_to_string(v, depth - 1) for k, v in value.items()}
elif is_sequence(value):
return strings.limit(value2json(value), LOG_STRING_LENGTH)
elif isinstance(value, number_types):
return value
elif is_text(value):
return strings.limit(value, LOG_STRING_LENGTH)
elif is_binary(value):
return strings.limit(bytes2base64(value), LOG_STRING_LENGTH)
elif isinstance(value, (date, datetime)):
return datetime2unix(value)
else:
return strings.limit(value2json(value), LOG_STRING_LENGTH)
def value2url_param(value):
"""
:param value:
:return: ascii URL
"""
from mo_json import value2json, json2value
def _encode(value):
return "".join(_map2url[c] for c in value.encode("utf8"))
if value == None:
return None
if is_data(value):
value_ = to_data(value)
output = "&".join(
kk + "=" + vv
for k, v in sorted(value_.leaves(), key=lambda p: p[0])
for kk, vv in [(value2url_param(k), value2url_param(v))]
if vv or vv == 0
)
elif is_text(value):
try:
json2value(value)
output = _encode(value2json(value))
except Exception:
output = _encode(value)
elif is_binary(value):
output = "".join(_map2url[c] for c in value)
elif is_many(value):
output = ",".join(
vv for v in value for vv in [value2url_param(v)] if vv or vv == 0
)
else:
output = _encode(value2json(value))
return output
def pretty_json(value):
try:
if value is False:
return "false"
elif value is True:
return "true"
elif is_data(value):
try:
value = unwrap(value)
items = sort_using_key(value.items(), lambda r: r[0])
values = [
encode_basestring(k) + PRETTY_COLON + pretty_json(v)
for k, v in items if v != None
]
if not values:
return "{}"
elif len(values) == 1:
return "{" + values[0] + "}"
else:
return "{\n" + ",\n".join(indent(v)
for v in values) + "\n}"
except Exception as e:
from mo_logs import Log
from mo_math import OR
if OR(not is_text(k) for k in value.keys()):
Log.error("JSON must have string keys: {{keys}}:",
keys=[k for k in value.keys()],
cause=e)
Log.error("problem making dict pretty: keys={{keys}}:",
keys=[k for k in value.keys()],
cause=e)
elif value in (None, Null):
return "null"
elif value.__class__ in (binary_type, text):
if is_binary(value):
value = value.decode('utf8')
try:
if "\n" in value and value.strip():
return pretty_json({
"$concat": value.split("\n"),
"separator": "\n"
})
else:
return quote(value)
except Exception as e:
from mo_logs import Log
try:
Log.note(
"try explicit convert of string with length {{length}}",
length=len(value))
acc = [QUOTE]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception:
c2 = c
c3 = text(c2)
acc.append(c3)
except BaseException:
pass
# Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g)
acc.append(QUOTE)
output = u"".join(acc)
Log.note("return value of length {{length}}",
length=len(output))
return output
except BaseException as f:
Log.warning("can not convert {{type}} to json",
type=f.__class__.__name__,
cause=f)
return "null"
elif is_list(value):
if not value:
return "[]"
if ARRAY_MAX_COLUMNS == 1:
return "[\n" + ",\n".join(
[indent(pretty_json(v)) for v in value]) + "\n]"
if len(value) == 1:
j = pretty_json(value[0])
if j.find("\n") >= 0:
return "[\n" + indent(j) + "\n]"
else:
return "[" + j + "]"
js = [pretty_json(v) for v in value]
max_len = max(*[len(j) for j in js])
if max_len <= ARRAY_ITEM_MAX_LENGTH and max(
*[j.find("\n") for j in js]) == -1:
# ALL TINY VALUES
num_columns = max(
1,
min(
ARRAY_MAX_COLUMNS,
int(
floor((ARRAY_ROW_LENGTH + 2.0) /
float(max_len +
2))))) # +2 TO COMPENSATE FOR COMMAS
if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW
return "[" + PRETTY_COMMA.join(js) + "]"
if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN
return "[\n" + ",\n".join(
[indent(pretty_json(v)) for v in value]) + "\n]"
content = ",\n".join(
PRETTY_COMMA.join(
j.rjust(max_len) for j in js[r:r + num_columns])
for r in xrange(0, len(js), num_columns))
return "[\n" + indent(content) + "\n]"
pretty_list = js
output = ["[\n"]
for i, p in enumerate(pretty_list):
try:
if i > 0:
output.append(",\n")
output.append(indent(p))
except Exception:
from mo_logs import Log
Log.warning(
"problem concatenating string of length {{len1}} and {{len2}}",
len1=len("".join(output)),
len2=len(p))
output.append("\n]")
try:
return "".join(output)
except Exception as e:
from mo_logs import Log
Log.error("not expected", cause=e)
elif hasattr(value, '__data__'):
d = value.__data__()
return pretty_json(d)
elif hasattr(value, '__json__'):
j = value.__json__()
if j == None:
return " null " # TODO: FIND OUT WHAT CAUSES THIS
return pretty_json(json_decoder(j))
elif scrub(value) is None:
return "null"
elif hasattr(value, '__iter__'):
return pretty_json(list(value))
elif hasattr(value, '__call__'):
return "null"
else:
try:
if int(value) == value:
return text(int(value))
except Exception:
pass
try:
if float(value) == value:
return text(float(value))
except Exception:
pass
return pypy_json_encode(value)
except Exception as e:
problem_serializing(value, e)
def pretty_json(value):
try:
if value is False:
return "false"
elif value is True:
return "true"
elif is_data(value):
try:
items = sort_using_key(value.items(), lambda r: r[0])
values = [encode_basestring(k) + PRETTY_COLON + pretty_json(v) for k, v in items if v != None]
if not values:
return "{}"
elif len(values) == 1:
return "{" + values[0] + "}"
else:
return "{\n" + ",\n".join(indent(v) for v in values) + "\n}"
except Exception as e:
from mo_logs import Log
from mo_math import OR
if OR(not is_text(k) for k in value.keys()):
Log.error(
"JSON must have string keys: {{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
Log.error(
"problem making dict pretty: keys={{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
elif value in (None, Null):
return "null"
elif value.__class__ in (binary_type, text_type):
if is_binary(value):
value = utf82unicode(value)
try:
return quote(value)
except Exception as e:
from mo_logs import Log
try:
Log.note("try explicit convert of string with length {{length}}", length=len(value))
acc = [QUOTE]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception:
c2 = c
c3 = text_type(c2)
acc.append(c3)
except BaseException:
pass
# Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g)
acc.append(QUOTE)
output = u"".join(acc)
Log.note("return value of length {{length}}", length=len(output))
return output
except BaseException as f:
Log.warning("can not convert {{type}} to json", type=f.__class__.__name__, cause=f)
return "null"
elif is_list(value):
if not value:
return "[]"
if ARRAY_MAX_COLUMNS == 1:
return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]"
if len(value) == 1:
j = pretty_json(value[0])
if j.find("\n") >= 0:
return "[\n" + indent(j) + "\n]"
else:
return "[" + j + "]"
js = [pretty_json(v) for v in value]
max_len = max(*[len(j) for j in js])
if max_len <= ARRAY_ITEM_MAX_LENGTH and max(*[j.find("\n") for j in js]) == -1:
# ALL TINY VALUES
num_columns = max(1, min(ARRAY_MAX_COLUMNS, int(floor((ARRAY_ROW_LENGTH + 2.0) / float(max_len + 2))))) # +2 TO COMPENSATE FOR COMMAS
if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW
return "[" + PRETTY_COMMA.join(js) + "]"
if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN
return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]"
content = ",\n".join(
PRETTY_COMMA.join(j.rjust(max_len) for j in js[r:r + num_columns])
for r in xrange(0, len(js), num_columns)
)
return "[\n" + indent(content) + "\n]"
pretty_list = js
output = ["[\n"]
for i, p in enumerate(pretty_list):
try:
if i > 0:
output.append(",\n")
output.append(indent(p))
except Exception:
from mo_logs import Log
Log.warning("problem concatenating string of length {{len1}} and {{len2}}",
len1=len("".join(output)),
len2=len(p)
)
output.append("\n]")
try:
return "".join(output)
except Exception as e:
from mo_logs import Log
Log.error("not expected", cause=e)
elif hasattr(value, '__data__'):
d = value.__data__()
return pretty_json(d)
elif hasattr(value, '__json__'):
j = value.__json__()
if j == None:
return " null " # TODO: FIND OUT WHAT CAUSES THIS
return pretty_json(json_decoder(j))
elif scrub(value) is None:
return "null"
elif hasattr(value, '__iter__'):
return pretty_json(list(value))
elif hasattr(value, '__call__'):
return "null"
else:
try:
if int(value) == value:
return text_type(int(value))
except Exception:
pass
try:
if float(value) == value:
return text_type(float(value))
except Exception:
pass
return pypy_json_encode(value)
except Exception as e:
problem_serializing(value, e)
def _scrub(value, is_done, stack, scrub_text, scrub_number):
if FIND_LOOPS:
_id = id(value)
if _id in stack:
Log.error("loop in JSON")
stack = stack + [_id]
type_ = value.__class__
if type_ in (none_type, NullType):
return None
elif type_ is text_type:
return scrub_text(value)
elif type_ is float:
if math.isnan(value) or math.isinf(value):
return None
return scrub_number(value)
elif type_ is bool:
return value
elif type_ in integer_types:
return scrub_number(value)
elif type_ in (date, datetime):
return scrub_number(datetime2unix(value))
elif type_ is timedelta:
return value.total_seconds()
elif type_ is Date:
return scrub_number(value.unix)
elif type_ is Duration:
return scrub_number(value.seconds)
elif type_ is str:
return utf82unicode(value)
elif type_ is Decimal:
return scrub_number(value)
elif type_ is Data:
return _scrub(_get(value, SLOT), is_done, stack, scrub_text,
scrub_number)
elif is_data(value):
_id = id(value)
if _id in is_done:
Log.warning("possible loop in structure detected")
return '"<LOOP IN STRUCTURE>"'
is_done.add(_id)
output = {}
for k, v in value.items():
if is_text(k):
pass
elif is_binary(k):
k = k.decode('utf8')
# elif hasattr(k, "__unicode__"):
# k = text_type(k)
else:
Log.error("keys must be strings")
v = _scrub(v, is_done, stack, scrub_text, scrub_number)
if v != None or is_data(v):
output[k] = v
is_done.discard(_id)
return output
elif type_ in (tuple, list, FlatList):
output = []
for v in value:
v = _scrub(v, is_done, stack, scrub_text, scrub_number)
output.append(v)
return output # if output else None
elif type_ is type:
return value.__name__
elif type_.__name__ == "bool_": # DEAR ME! Numpy has it's own booleans (value==False could be used, but 0==False in Python. DOH!)
if value == False:
return False
else:
return True
elif not isinstance(value, Except) and isinstance(value, Exception):
return _scrub(Except.wrap(value), is_done, stack, scrub_text,
scrub_number)
elif hasattr(value, '__data__'):
try:
return _scrub(value.__data__(), is_done, stack, scrub_text,
scrub_number)
except Exception as e:
Log.error("problem with calling __json__()", e)
elif hasattr(value, 'co_code') or hasattr(value, "f_locals"):
return None
elif hasattr(value, '__iter__'):
output = []
for v in value:
v = _scrub(v, is_done, stack, scrub_text, scrub_number)
output.append(v)
return output
elif hasattr(value, '__call__'):
return text_type(repr(value))
else:
return _scrub(DataObject(value), is_done, stack, scrub_text,
scrub_number)