def openObject(self, name, attributes=None):
'''
Used to open a JSON object.
'''
assert isinstance(name, str), 'Invalid name %s' % name
assert attributes is None or isinstance(
attributes, dict), 'Invalid attributes %s' % attributes
out = self.out
if not self.isFirst: out.write(',')
if self.isObject and self.isObject[0]:
out.write(encode_basestring(name))
out.write(':')
out.write('{')
self.isFirst = True
if attributes:
for attrName, attrValue in attributes.items():
assert isinstance(attrName,
str), 'Invalid attribute name %s' % attrName
assert isinstance(
attrValue, str), 'Invalid attribute value %s' % attrValue
if self.isFirst: self.isFirst = False
else: out.write(',')
out.write(encode_basestring(attrName))
out.write(':')
out.write(encode_basestring(attrValue))
def openObject(self, name, attributes=None):
"""
Used to open a JSON object.
"""
assert isinstance(name, str), "Invalid name %s" % name
assert attributes is None or isinstance(attributes, dict), "Invalid attributes %s" % attributes
out = self.out
if not self.isFirst:
out.write(",")
if self.isObject and self.isObject[0]:
out.write(encode_basestring(name))
out.write(":")
out.write("{")
self.isFirst = True
if attributes:
for attrName, attrValue in attributes.items():
assert isinstance(attrName, str), "Invalid attribute name %s" % attrName
assert isinstance(attrValue, str), "Invalid attribute value %s" % attrValue
if self.isFirst:
self.isFirst = False
else:
out.write(",")
out.write(encode_basestring(attrName))
out.write(":")
out.write(encode_basestring(attrValue))
def _dict2json(value, sub_schema, path, net_new_properties, buffer):
prefix = '{'
for k, v in sort_using_key(value.items(), lambda r: r[0]):
if v == None or v == '':
continue
append(buffer, prefix)
prefix = COMMA
if is_binary(k):
k = k.decode('utf8')
if not is_text(k):
Log.error("Expecting property name to be a string")
if k not in sub_schema:
sub_schema[k] = {}
net_new_properties.append(path + [k])
append(buffer, encode_basestring(encode_property(k)))
append(buffer, COLON)
typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer)
if prefix is COMMA:
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
else:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
def encode(self, o):
"""Return a JSON string representation of a Python data structure.
>>> JSONEncoder().encode({"foo": ["bar", "baz"]})
'{"foo": ["bar", "baz"]}'
"""
# This is for extremely simple cases and benchmarks.
if isinstance(o, six.string_types):
if isinstance(o, str):
_encoding = self.encoding
if (_encoding is not None and not (_encoding == 'utf-8')):
o = o.decode(_encoding)
if self.ensure_ascii:
return encode_basestring_ascii(o)
else:
return encode_basestring(o)
# This doesn't pass the iterator directly to ''.join() because the
# exceptions aren't as detailed. The list call should be roughly
# equivalent to the PySequence_Fast that ''.join() would do.
chunks = self.iterencode(o, _one_shot=True)
if not isinstance(chunks, (list, tuple)):
chunks = list(chunks)
out = ''.join(chunks)
out = re.sub(fix_key, r'\1 =', out) # libconfig's name
out = re.sub(fix_hex, r'\1', out) # libconfig's hex
out = re.sub(fix_hex, r'\1', out) # libconfig's hex
return out[1:-1] # remove outter {}
def _dict2json(value, sub_schema, path, net_new_properties, buffer):
prefix = '{'
for k, v in sort_using_key(value.items(), lambda r: r[0]):
if v == None or v == '':
continue
append(buffer, prefix)
prefix = COMMA
if is_binary(k):
k = utf82unicode(k)
if not is_text(k):
Log.error("Expecting property name to be a string")
if k not in sub_schema:
sub_schema[k] = {}
net_new_properties.append(path + [k])
append(buffer, encode_basestring(encode_property(k)))
append(buffer, COLON)
typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer)
if prefix is COMMA:
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
else:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
def encode(self, o):
"""Return a JSON string representation of a Python data structure.
>>> JSONEncoder().encode({"foo": ["bar", "baz"]})
'{"foo": ["bar", "baz"]}'
"""
# This is for extremely simple cases and benchmarks.
if isinstance(o, basestring):
if isinstance(o, str):
_encoding = self.encoding
if (_encoding is not None
and not (_encoding == 'utf-8')):
o = o.decode(_encoding)
if self.ensure_ascii:
return encode_basestring_ascii(o)
else:
return encode_basestring(o)
# This doesn't pass the iterator directly to ''.join() because the
# exceptions aren't as detailed. The list call should be roughly
# equivalent to the PySequence_Fast that ''.join() would do.
chunks = self.iterencode(o, {}, _one_shot=True)
if not isinstance(chunks, (list, tuple)):
chunks = list(chunks)
return ''.join(chunks)
def quote(value):
if value == None:
output = ""
elif isinstance(value, text_type):
output = encode_basestring(value)
else:
output = _json.dumps(value)
return output
def value(self, name, value):
'''
@see: IRender.value
'''
assert self.isObject, 'No container for value'
assert isinstance(name, str), 'Invalid name %s' % name
assert isinstance(value, str), 'Invalid value %s' % value
out = self.out
if self.isFirst: self.isFirst = False
else: out.write(',')
if self.isObject[0]:
out.write(encode_basestring(name))
out.write(':')
out.write(encode_basestring(value))
else:
out.write(encode_basestring(value))
def encode(value):
'''
Encodes the value as a JSON value.
'''
if isinstance(value, str): return encode_basestring(value)
if value is True: return 'true'
if value is False: return 'false'
if isinstance(value, float): return repr(value)
assert isinstance(value, int), 'Invalid value %s' % value
return str(value)
def value(self, name, value):
"""
@see: IRender.value
"""
assert self.isObject, "No container for value"
assert isinstance(name, str), "Invalid name %s" % name
assert isinstance(value, str), "Invalid value %s" % value
out = self.out
if self.isFirst:
self.isFirst = False
else:
out.write(",")
if self.isObject[0]:
out.write(encode_basestring(name))
out.write(":")
out.write(encode_basestring(value))
else:
out.write(encode_basestring(value))
def encode(value):
'''
Encodes the value as a JSON value.
'''
if isinstance(value, str): return encode_basestring(value)
if value is True: return 'true'
if value is False: return 'false'
if isinstance(value, float): return repr(value)
assert isinstance(value, int), 'Invalid value %s' % value
return str(value)
def quote(value):
"""
return JSON-quoted value
:param value:
:return:
"""
if value == None:
output = ""
elif is_text(value):
output = encode_basestring(value)
else:
output = _json.dumps(value)
return output
def collectionStart(self, name, attributes=None):
'''
@see: IRender.collectionStart
'''
assert isinstance(name, str), 'Invalid name %s' % name
out = self.out
self.openObject(name, attributes)
if not self.isFirst: out.write(',')
out.write(encode_basestring(name))
out.write(':[')
self.isFirst = True
self.isObject.appendleft(False)
def quote(value):
"""
return JSON-quoted value
:param value:
:return:
"""
if value == None:
output = ""
elif isinstance(value, text_type):
output = encode_basestring(value)
else:
output = _json.dumps(value)
return output
def collectionStart(self, name, attributes=None):
"""
@see: IRender.collectionStart
"""
assert isinstance(name, str), "Invalid name %s" % name
out = self.out
self.openObject(name, attributes)
if not self.isFirst:
out.write(",")
out.write(encode_basestring(name))
out.write(":[")
self.isFirst = True
self.isObject.appendleft(False)
def visit_string(self, node):
return encode_basestring(node.value)
def visit_string(self, node):
return encode_basestring(node.value)
def pretty_json(value):
try:
if value is False:
return "false"
elif value is True:
return "true"
elif isinstance(value, Mapping):
try:
items = sort_using_key(list(value.items()), lambda r: r[0])
values = [
encode_basestring(k) + PRETTY_COLON +
indent(pretty_json(v)).strip() for k, v in items
if v != None
]
if not values:
return "{}"
elif len(values) == 1:
return "{" + values[0] + "}"
else:
return "{\n" + INDENT + (",\n" +
INDENT).join(values) + "\n}"
except Exception as e:
from mo_logs import Log
from mo_math import OR
if OR(not isinstance(k, text_type) for k in value.keys()):
Log.error("JSON must have string keys: {{keys}}:",
keys=[k for k in value.keys()],
cause=e)
Log.error("problem making dict pretty: keys={{keys}}:",
keys=[k for k in value.keys()],
cause=e)
elif value in (None, Null):
return "null"
elif isinstance(value, (text_type, binary_type)):
if isinstance(value, binary_type):
value = utf82unicode(value)
try:
return quote(value)
except Exception as e:
from mo_logs import Log
try:
Log.note(
"try explicit convert of string with length {{length}}",
length=len(value))
acc = [QUOTE]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception:
c2 = c
c3 = text_type(c2)
acc.append(c3)
except BaseException:
pass
# Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g)
acc.append(QUOTE)
output = u"".join(acc)
Log.note("return value of length {{length}}",
length=len(output))
return output
except BaseException as f:
Log.warning("can not even explicit convert {{type}}",
type=f.__class__.__name__,
cause=f)
return "null"
elif isinstance(value, list):
if not value:
return "[]"
if ARRAY_MAX_COLUMNS == 1:
return "[\n" + ",\n".join(
[indent(pretty_json(v)) for v in value]) + "\n]"
if len(value) == 1:
j = pretty_json(value[0])
if j.find("\n") >= 0:
return "[\n" + indent(j) + "\n]"
else:
return "[" + j + "]"
js = [pretty_json(v) for v in value]
max_len = max(*[len(j) for j in js])
if max_len <= ARRAY_ITEM_MAX_LENGTH and max(
*[j.find("\n") for j in js]) == -1:
# ALL TINY VALUES
num_columns = max(
1,
min(
ARRAY_MAX_COLUMNS,
int(
floor((ARRAY_ROW_LENGTH + 2.0) /
float(max_len +
2))))) # +2 TO COMPENSATE FOR COMMAS
if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW
return "[" + PRETTY_COMMA.join(js) + "]"
if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN
return "[\n" + ",\n".join(
[indent(pretty_json(v)) for v in value]) + "\n]"
content = ",\n".join(
PRETTY_COMMA.join(
j.rjust(max_len) for j in js[r:r + num_columns])
for r in xrange(0, len(js), num_columns))
return "[\n" + indent(content) + "\n]"
pretty_list = js
output = ["[\n"]
for i, p in enumerate(pretty_list):
try:
if i > 0:
output.append(",\n")
output.append(indent(p))
except Exception:
from mo_logs import Log
Log.warning(
"problem concatenating string of length {{len1}} and {{len2}}",
len1=len("".join(output)),
len2=len(p))
output.append("\n]")
try:
return "".join(output)
except Exception as e:
from mo_logs import Log
Log.error("not expected", cause=e)
elif hasattr(value, '__data__'):
d = value.__data__()
return pretty_json(d)
elif hasattr(value, '__json__'):
j = value.__json__()
if j == None:
return " null " # TODO: FIND OUT WHAT CAUSES THIS
return pretty_json(json_decoder(j))
elif scrub(value) is None:
return "null"
elif hasattr(value, '__iter__'):
return pretty_json(list(value))
elif hasattr(value, '__call__'):
return "null"
else:
try:
if int(value) == value:
return text_type(int(value))
except Exception:
pass
try:
if float(value) == value:
return text_type(float(value))
except Exception:
pass
return pypy_json_encode(value)
except Exception as e:
problem_serializing(value, e)
def _str(value):
return js.String(encode_basestring(value))
def brief_encode_basestring(s):
text = encode_basestring(s)
text = text[:HRDjangoJSONEncoder.MAX_ITEM_LENGTH - 3] + '...' \
if len(text) > HRDjangoJSONEncoder.MAX_ITEM_LENGTH else text
return json.dumps(text)
def pretty_json(value):
try:
if value is False:
return "false"
elif value is True:
return "true"
elif is_data(value):
try:
items = sort_using_key(value.items(), lambda r: r[0])
values = [encode_basestring(k) + PRETTY_COLON + pretty_json(v) for k, v in items if v != None]
if not values:
return "{}"
elif len(values) == 1:
return "{" + values[0] + "}"
else:
return "{\n" + ",\n".join(indent(v) for v in values) + "\n}"
except Exception as e:
from mo_logs import Log
from mo_math import OR
if OR(not is_text(k) for k in value.keys()):
Log.error(
"JSON must have string keys: {{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
Log.error(
"problem making dict pretty: keys={{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
elif value in (None, Null):
return "null"
elif value.__class__ in (binary_type, text_type):
if is_binary(value):
value = utf82unicode(value)
try:
return quote(value)
except Exception as e:
from mo_logs import Log
try:
Log.note("try explicit convert of string with length {{length}}", length=len(value))
acc = [QUOTE]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception:
c2 = c
c3 = text_type(c2)
acc.append(c3)
except BaseException:
pass
# Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g)
acc.append(QUOTE)
output = u"".join(acc)
Log.note("return value of length {{length}}", length=len(output))
return output
except BaseException as f:
Log.warning("can not convert {{type}} to json", type=f.__class__.__name__, cause=f)
return "null"
elif is_list(value):
if not value:
return "[]"
if ARRAY_MAX_COLUMNS == 1:
return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]"
if len(value) == 1:
j = pretty_json(value[0])
if j.find("\n") >= 0:
return "[\n" + indent(j) + "\n]"
else:
return "[" + j + "]"
js = [pretty_json(v) for v in value]
max_len = max(*[len(j) for j in js])
if max_len <= ARRAY_ITEM_MAX_LENGTH and max(*[j.find("\n") for j in js]) == -1:
# ALL TINY VALUES
num_columns = max(1, min(ARRAY_MAX_COLUMNS, int(floor((ARRAY_ROW_LENGTH + 2.0) / float(max_len + 2))))) # +2 TO COMPENSATE FOR COMMAS
if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW
return "[" + PRETTY_COMMA.join(js) + "]"
if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN
return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]"
content = ",\n".join(
PRETTY_COMMA.join(j.rjust(max_len) for j in js[r:r + num_columns])
for r in xrange(0, len(js), num_columns)
)
return "[\n" + indent(content) + "\n]"
pretty_list = js
output = ["[\n"]
for i, p in enumerate(pretty_list):
try:
if i > 0:
output.append(",\n")
output.append(indent(p))
except Exception:
from mo_logs import Log
Log.warning("problem concatenating string of length {{len1}} and {{len2}}",
len1=len("".join(output)),
len2=len(p)
)
output.append("\n]")
try:
return "".join(output)
except Exception as e:
from mo_logs import Log
Log.error("not expected", cause=e)
elif hasattr(value, '__data__'):
d = value.__data__()
return pretty_json(d)
elif hasattr(value, '__json__'):
j = value.__json__()
if j == None:
return " null " # TODO: FIND OUT WHAT CAUSES THIS
return pretty_json(json_decoder(j))
elif scrub(value) is None:
return "null"
elif hasattr(value, '__iter__'):
return pretty_json(list(value))
elif hasattr(value, '__call__'):
return "null"
else:
try:
if int(value) == value:
return text_type(int(value))
except Exception:
pass
try:
if float(value) == value:
return text_type(float(value))
except Exception:
pass
return pypy_json_encode(value)
except Exception as e:
problem_serializing(value, e)
def processConfig(configlines,
with_comments=False,
verbose=False,
substitutions=None):
"""
Process the "raw" config lines from stdconfig.py into a JSON object
(a Python L{dict}) that is ordered and contains commentary based on
the Python comments.
@param configlines: config data lines
@type configlines: L{list} of L{str}
@param with_comments: whether to include comments or not
@type with_comments: L{bool}
@param verbose: print out intermediate state
@type verbose: L{bool}
@return: the serialized JSON object
@rtype: L{OrderedDict}
"""
# Comments will either be "block" (as in section dividers) or "inline"
# (as in appended to the end of the line). We treat these slightly
# differently wrt to whitespace and where they appear.
lines = []
ctr = 0
block_comment = []
inline_comment = []
# Regular expression to match an inline comment and a
# value containing a numeric expression that needs to be
# evaluated (e.g. "60 * 60")
comments = re.compile("([ ]*.*?,?)[ ]*#[ ]*(.*)[ ]*$")
value = re.compile("([^:]+:[ ]+)([0-9 \*]+)(.*)")
for line in configlines.splitlines():
if line.strip() and line.strip()[0] == "#":
# Line with just a comment is a block comment unless the
# previous comment was inline (in which case it is a multi-line
# inline). Aggregate block and inline comments into one overall
# comment.
comment = line.strip()[1:].strip()
if len(comment) == 0 and len(block_comment) == 0 and len(
inline_comment) == 0:
pass
elif inline_comment:
inline_comment.append(comment if comment else "\n")
else:
block_comment.append(comment if comment else "\n")
continue
elif block_comment:
# Generate a block comment JSON member
if with_comments:
comment_type = "comment_" if line.strip(
) and block_comment[-1] != "\n" else "section_"
while block_comment[-1] == "\n":
block_comment.pop()
lines.append("\"{}{}\": {},".format(
comment_type, ctr,
encode_basestring(" ".join(block_comment))))
ctr += 1
block_comment = []
elif inline_comment:
# Generate an inline comment JSON member
if with_comments:
lines.insert(
-1, "\"comment_{}\": {},".format(
ctr, encode_basestring(" ".join(inline_comment))))
ctr += 1
inline_comment = []
# Check if the current line contains an inline comment, if so extract
# the comment and add to the current inline comments list
m = comments.match(line)
if m:
inline_comment.append(m.group(2))
append = m.group(1)
else:
append = line
# Do some simple value conversions
append = append.rstrip().replace(" None", ' ""').replace(
" True", " true").replace(" False",
" false").replace("\\", "\\\\")
# Look for special substitutions
if substitutions:
for subskey in substitutions.keys():
pos = append.find(subskey)
if pos >= 0:
actual = append[pos + len(subskey) + 2:]
comma = ""
if actual[-1] == ",":
actual = actual[:-1]
comma = ","
actual = actual[:-2]
append = "{}{}{}".format(
append[:pos],
json.dumps(substitutions[subskey][actual]),
comma,
)
break
# Look for numeric expressions in the value and eval() those to get a value
# that is compatible with JSON
m = value.match(append)
if m:
expression = eval(m.group(2))
append = "{}{}{}".format(m.group(1), expression, m.group(3))
# Remove trailing commas for the last items in an array
# or object as JSON does not like that
if append.strip() and append.strip()[0] in ("]", "}"):
if lines[-1][-1] == ",":
lines[-1] = lines[-1][:-1]
# Line is ready to use
lines.append(append)
newj = "\n".join(lines)
if verbose:
print(newj)
# Created an ordered JSON object
j = json.loads(newj, object_pairs_hook=OrderedDict)
return j
