def openObject(self, name, attributes=None): ''' Used to open a JSON object. ''' assert isinstance(name, str), 'Invalid name %s' % name assert attributes is None or isinstance( attributes, dict), 'Invalid attributes %s' % attributes out = self.out if not self.isFirst: out.write(',') if self.isObject and self.isObject[0]: out.write(encode_basestring(name)) out.write(':') out.write('{') self.isFirst = True if attributes: for attrName, attrValue in attributes.items(): assert isinstance(attrName, str), 'Invalid attribute name %s' % attrName assert isinstance( attrValue, str), 'Invalid attribute value %s' % attrValue if self.isFirst: self.isFirst = False else: out.write(',') out.write(encode_basestring(attrName)) out.write(':') out.write(encode_basestring(attrValue))
def openObject(self, name, attributes=None): """ Used to open a JSON object. """ assert isinstance(name, str), "Invalid name %s" % name assert attributes is None or isinstance(attributes, dict), "Invalid attributes %s" % attributes out = self.out if not self.isFirst: out.write(",") if self.isObject and self.isObject[0]: out.write(encode_basestring(name)) out.write(":") out.write("{") self.isFirst = True if attributes: for attrName, attrValue in attributes.items(): assert isinstance(attrName, str), "Invalid attribute name %s" % attrName assert isinstance(attrValue, str), "Invalid attribute value %s" % attrValue if self.isFirst: self.isFirst = False else: out.write(",") out.write(encode_basestring(attrName)) out.write(":") out.write(encode_basestring(attrValue))
def _dict2json(value, sub_schema, path, net_new_properties, buffer): prefix = '{' for k, v in sort_using_key(value.items(), lambda r: r[0]): if v == None or v == '': continue append(buffer, prefix) prefix = COMMA if is_binary(k): k = k.decode('utf8') if not is_text(k): Log.error("Expecting property name to be a string") if k not in sub_schema: sub_schema[k] = {} net_new_properties.append(path + [k]) append(buffer, encode_basestring(encode_property(k))) append(buffer, COLON) typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer) if prefix is COMMA: append(buffer, COMMA) append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}') else: append(buffer, '{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}')
def encode(self, o): """Return a JSON string representation of a Python data structure. >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) '{"foo": ["bar", "baz"]}' """ # This is for extremely simple cases and benchmarks. if isinstance(o, six.string_types): if isinstance(o, str): _encoding = self.encoding if (_encoding is not None and not (_encoding == 'utf-8')): o = o.decode(_encoding) if self.ensure_ascii: return encode_basestring_ascii(o) else: return encode_basestring(o) # This doesn't pass the iterator directly to ''.join() because the # exceptions aren't as detailed. The list call should be roughly # equivalent to the PySequence_Fast that ''.join() would do. chunks = self.iterencode(o, _one_shot=True) if not isinstance(chunks, (list, tuple)): chunks = list(chunks) out = ''.join(chunks) out = re.sub(fix_key, r'\1 =', out) # libconfig's name out = re.sub(fix_hex, r'\1', out) # libconfig's hex out = re.sub(fix_hex, r'\1', out) # libconfig's hex return out[1:-1] # remove outter {}
def _dict2json(value, sub_schema, path, net_new_properties, buffer): prefix = '{' for k, v in sort_using_key(value.items(), lambda r: r[0]): if v == None or v == '': continue append(buffer, prefix) prefix = COMMA if is_binary(k): k = utf82unicode(k) if not is_text(k): Log.error("Expecting property name to be a string") if k not in sub_schema: sub_schema[k] = {} net_new_properties.append(path + [k]) append(buffer, encode_basestring(encode_property(k))) append(buffer, COLON) typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer) if prefix is COMMA: append(buffer, COMMA) append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}') else: append(buffer, '{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}')
def encode(self, o): """Return a JSON string representation of a Python data structure. >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) '{"foo": ["bar", "baz"]}' """ # This is for extremely simple cases and benchmarks. if isinstance(o, basestring): if isinstance(o, str): _encoding = self.encoding if (_encoding is not None and not (_encoding == 'utf-8')): o = o.decode(_encoding) if self.ensure_ascii: return encode_basestring_ascii(o) else: return encode_basestring(o) # This doesn't pass the iterator directly to ''.join() because the # exceptions aren't as detailed. The list call should be roughly # equivalent to the PySequence_Fast that ''.join() would do. chunks = self.iterencode(o, {}, _one_shot=True) if not isinstance(chunks, (list, tuple)): chunks = list(chunks) return ''.join(chunks)
def quote(value): if value == None: output = "" elif isinstance(value, text_type): output = encode_basestring(value) else: output = _json.dumps(value) return output
def value(self, name, value): ''' @see: IRender.value ''' assert self.isObject, 'No container for value' assert isinstance(name, str), 'Invalid name %s' % name assert isinstance(value, str), 'Invalid value %s' % value out = self.out if self.isFirst: self.isFirst = False else: out.write(',') if self.isObject[0]: out.write(encode_basestring(name)) out.write(':') out.write(encode_basestring(value)) else: out.write(encode_basestring(value))
def encode(value): ''' Encodes the value as a JSON value. ''' if isinstance(value, str): return encode_basestring(value) if value is True: return 'true' if value is False: return 'false' if isinstance(value, float): return repr(value) assert isinstance(value, int), 'Invalid value %s' % value return str(value)
def value(self, name, value): """ @see: IRender.value """ assert self.isObject, "No container for value" assert isinstance(name, str), "Invalid name %s" % name assert isinstance(value, str), "Invalid value %s" % value out = self.out if self.isFirst: self.isFirst = False else: out.write(",") if self.isObject[0]: out.write(encode_basestring(name)) out.write(":") out.write(encode_basestring(value)) else: out.write(encode_basestring(value))
def encode(value): ''' Encodes the value as a JSON value. ''' if isinstance(value, str): return encode_basestring(value) if value is True: return 'true' if value is False: return 'false' if isinstance(value, float): return repr(value) assert isinstance(value, int), 'Invalid value %s' % value return str(value)
def quote(value): """ return JSON-quoted value :param value: :return: """ if value == None: output = "" elif is_text(value): output = encode_basestring(value) else: output = _json.dumps(value) return output
def collectionStart(self, name, attributes=None): ''' @see: IRender.collectionStart ''' assert isinstance(name, str), 'Invalid name %s' % name out = self.out self.openObject(name, attributes) if not self.isFirst: out.write(',') out.write(encode_basestring(name)) out.write(':[') self.isFirst = True self.isObject.appendleft(False)
def quote(value): """ return JSON-quoted value :param value: :return: """ if value == None: output = "" elif isinstance(value, text_type): output = encode_basestring(value) else: output = _json.dumps(value) return output
def collectionStart(self, name, attributes=None): """ @see: IRender.collectionStart """ assert isinstance(name, str), "Invalid name %s" % name out = self.out self.openObject(name, attributes) if not self.isFirst: out.write(",") out.write(encode_basestring(name)) out.write(":[") self.isFirst = True self.isObject.appendleft(False)
def visit_string(self, node): return encode_basestring(node.value)
def visit_string(self, node): return encode_basestring(node.value)
def pretty_json(value): try: if value is False: return "false" elif value is True: return "true" elif isinstance(value, Mapping): try: items = sort_using_key(list(value.items()), lambda r: r[0]) values = [ encode_basestring(k) + PRETTY_COLON + indent(pretty_json(v)).strip() for k, v in items if v != None ] if not values: return "{}" elif len(values) == 1: return "{" + values[0] + "}" else: return "{\n" + INDENT + (",\n" + INDENT).join(values) + "\n}" except Exception as e: from mo_logs import Log from mo_math import OR if OR(not isinstance(k, text_type) for k in value.keys()): Log.error("JSON must have string keys: {{keys}}:", keys=[k for k in value.keys()], cause=e) Log.error("problem making dict pretty: keys={{keys}}:", keys=[k for k in value.keys()], cause=e) elif value in (None, Null): return "null" elif isinstance(value, (text_type, binary_type)): if isinstance(value, binary_type): value = utf82unicode(value) try: return quote(value) except Exception as e: from mo_logs import Log try: Log.note( "try explicit convert of string with length {{length}}", length=len(value)) acc = [QUOTE] for c in value: try: try: c2 = ESCAPE_DCT[c] except Exception: c2 = c c3 = text_type(c2) acc.append(c3) except BaseException: pass # Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g) acc.append(QUOTE) output = u"".join(acc) Log.note("return value of length {{length}}", length=len(output)) return output except BaseException as f: Log.warning("can not even explicit convert {{type}}", type=f.__class__.__name__, cause=f) return "null" elif isinstance(value, list): if not value: return "[]" if ARRAY_MAX_COLUMNS == 1: return "[\n" + ",\n".join( [indent(pretty_json(v)) for v in value]) + "\n]" if len(value) == 1: j = pretty_json(value[0]) if j.find("\n") >= 0: return "[\n" + indent(j) + "\n]" else: return "[" + j + "]" js = [pretty_json(v) for v in value] max_len = max(*[len(j) for j in js]) if max_len <= ARRAY_ITEM_MAX_LENGTH and max( *[j.find("\n") for j in js]) == -1: # ALL TINY VALUES num_columns = max( 1, min( ARRAY_MAX_COLUMNS, int( floor((ARRAY_ROW_LENGTH + 2.0) / float(max_len + 2))))) # +2 TO COMPENSATE FOR COMMAS if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW return "[" + PRETTY_COMMA.join(js) + "]" if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN return "[\n" + ",\n".join( [indent(pretty_json(v)) for v in value]) + "\n]" content = ",\n".join( PRETTY_COMMA.join( j.rjust(max_len) for j in js[r:r + num_columns]) for r in xrange(0, len(js), num_columns)) return "[\n" + indent(content) + "\n]" pretty_list = js output = ["[\n"] for i, p in enumerate(pretty_list): try: if i > 0: output.append(",\n") output.append(indent(p)) except Exception: from mo_logs import Log Log.warning( "problem concatenating string of length {{len1}} and {{len2}}", len1=len("".join(output)), len2=len(p)) output.append("\n]") try: return "".join(output) except Exception as e: from mo_logs import Log Log.error("not expected", cause=e) elif hasattr(value, '__data__'): d = value.__data__() return pretty_json(d) elif hasattr(value, '__json__'): j = value.__json__() if j == None: return " null " # TODO: FIND OUT WHAT CAUSES THIS return pretty_json(json_decoder(j)) elif scrub(value) is None: return "null" elif hasattr(value, '__iter__'): return pretty_json(list(value)) elif hasattr(value, '__call__'): return "null" else: try: if int(value) == value: return text_type(int(value)) except Exception: pass try: if float(value) == value: return text_type(float(value)) except Exception: pass return pypy_json_encode(value) except Exception as e: problem_serializing(value, e)
def _str(value): return js.String(encode_basestring(value))
def brief_encode_basestring(s): text = encode_basestring(s) text = text[:HRDjangoJSONEncoder.MAX_ITEM_LENGTH - 3] + '...' \ if len(text) > HRDjangoJSONEncoder.MAX_ITEM_LENGTH else text return json.dumps(text)
def pretty_json(value): try: if value is False: return "false" elif value is True: return "true" elif is_data(value): try: items = sort_using_key(value.items(), lambda r: r[0]) values = [encode_basestring(k) + PRETTY_COLON + pretty_json(v) for k, v in items if v != None] if not values: return "{}" elif len(values) == 1: return "{" + values[0] + "}" else: return "{\n" + ",\n".join(indent(v) for v in values) + "\n}" except Exception as e: from mo_logs import Log from mo_math import OR if OR(not is_text(k) for k in value.keys()): Log.error( "JSON must have string keys: {{keys}}:", keys=[k for k in value.keys()], cause=e ) Log.error( "problem making dict pretty: keys={{keys}}:", keys=[k for k in value.keys()], cause=e ) elif value in (None, Null): return "null" elif value.__class__ in (binary_type, text_type): if is_binary(value): value = utf82unicode(value) try: return quote(value) except Exception as e: from mo_logs import Log try: Log.note("try explicit convert of string with length {{length}}", length=len(value)) acc = [QUOTE] for c in value: try: try: c2 = ESCAPE_DCT[c] except Exception: c2 = c c3 = text_type(c2) acc.append(c3) except BaseException: pass # Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g) acc.append(QUOTE) output = u"".join(acc) Log.note("return value of length {{length}}", length=len(output)) return output except BaseException as f: Log.warning("can not convert {{type}} to json", type=f.__class__.__name__, cause=f) return "null" elif is_list(value): if not value: return "[]" if ARRAY_MAX_COLUMNS == 1: return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]" if len(value) == 1: j = pretty_json(value[0]) if j.find("\n") >= 0: return "[\n" + indent(j) + "\n]" else: return "[" + j + "]" js = [pretty_json(v) for v in value] max_len = max(*[len(j) for j in js]) if max_len <= ARRAY_ITEM_MAX_LENGTH and max(*[j.find("\n") for j in js]) == -1: # ALL TINY VALUES num_columns = max(1, min(ARRAY_MAX_COLUMNS, int(floor((ARRAY_ROW_LENGTH + 2.0) / float(max_len + 2))))) # +2 TO COMPENSATE FOR COMMAS if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW return "[" + PRETTY_COMMA.join(js) + "]" if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]" content = ",\n".join( PRETTY_COMMA.join(j.rjust(max_len) for j in js[r:r + num_columns]) for r in xrange(0, len(js), num_columns) ) return "[\n" + indent(content) + "\n]" pretty_list = js output = ["[\n"] for i, p in enumerate(pretty_list): try: if i > 0: output.append(",\n") output.append(indent(p)) except Exception: from mo_logs import Log Log.warning("problem concatenating string of length {{len1}} and {{len2}}", len1=len("".join(output)), len2=len(p) ) output.append("\n]") try: return "".join(output) except Exception as e: from mo_logs import Log Log.error("not expected", cause=e) elif hasattr(value, '__data__'): d = value.__data__() return pretty_json(d) elif hasattr(value, '__json__'): j = value.__json__() if j == None: return " null " # TODO: FIND OUT WHAT CAUSES THIS return pretty_json(json_decoder(j)) elif scrub(value) is None: return "null" elif hasattr(value, '__iter__'): return pretty_json(list(value)) elif hasattr(value, '__call__'): return "null" else: try: if int(value) == value: return text_type(int(value)) except Exception: pass try: if float(value) == value: return text_type(float(value)) except Exception: pass return pypy_json_encode(value) except Exception as e: problem_serializing(value, e)
def processConfig(configlines, with_comments=False, verbose=False, substitutions=None): """ Process the "raw" config lines from stdconfig.py into a JSON object (a Python L{dict}) that is ordered and contains commentary based on the Python comments. @param configlines: config data lines @type configlines: L{list} of L{str} @param with_comments: whether to include comments or not @type with_comments: L{bool} @param verbose: print out intermediate state @type verbose: L{bool} @return: the serialized JSON object @rtype: L{OrderedDict} """ # Comments will either be "block" (as in section dividers) or "inline" # (as in appended to the end of the line). We treat these slightly # differently wrt to whitespace and where they appear. lines = [] ctr = 0 block_comment = [] inline_comment = [] # Regular expression to match an inline comment and a # value containing a numeric expression that needs to be # evaluated (e.g. "60 * 60") comments = re.compile("([ ]*.*?,?)[ ]*#[ ]*(.*)[ ]*$") value = re.compile("([^:]+:[ ]+)([0-9 \*]+)(.*)") for line in configlines.splitlines(): if line.strip() and line.strip()[0] == "#": # Line with just a comment is a block comment unless the # previous comment was inline (in which case it is a multi-line # inline). Aggregate block and inline comments into one overall # comment. comment = line.strip()[1:].strip() if len(comment) == 0 and len(block_comment) == 0 and len( inline_comment) == 0: pass elif inline_comment: inline_comment.append(comment if comment else "\n") else: block_comment.append(comment if comment else "\n") continue elif block_comment: # Generate a block comment JSON member if with_comments: comment_type = "comment_" if line.strip( ) and block_comment[-1] != "\n" else "section_" while block_comment[-1] == "\n": block_comment.pop() lines.append("\"{}{}\": {},".format( comment_type, ctr, encode_basestring(" ".join(block_comment)))) ctr += 1 block_comment = [] elif inline_comment: # Generate an inline comment JSON member if with_comments: lines.insert( -1, "\"comment_{}\": {},".format( ctr, encode_basestring(" ".join(inline_comment)))) ctr += 1 inline_comment = [] # Check if the current line contains an inline comment, if so extract # the comment and add to the current inline comments list m = comments.match(line) if m: inline_comment.append(m.group(2)) append = m.group(1) else: append = line # Do some simple value conversions append = append.rstrip().replace(" None", ' ""').replace( " True", " true").replace(" False", " false").replace("\\", "\\\\") # Look for special substitutions if substitutions: for subskey in substitutions.keys(): pos = append.find(subskey) if pos >= 0: actual = append[pos + len(subskey) + 2:] comma = "" if actual[-1] == ",": actual = actual[:-1] comma = "," actual = actual[:-2] append = "{}{}{}".format( append[:pos], json.dumps(substitutions[subskey][actual]), comma, ) break # Look for numeric expressions in the value and eval() those to get a value # that is compatible with JSON m = value.match(append) if m: expression = eval(m.group(2)) append = "{}{}{}".format(m.group(1), expression, m.group(3)) # Remove trailing commas for the last items in an array # or object as JSON does not like that if append.strip() and append.strip()[0] in ("]", "}"): if lines[-1][-1] == ",": lines[-1] = lines[-1][:-1] # Line is ready to use lines.append(append) newj = "\n".join(lines) if verbose: print(newj) # Created an ordered JSON object j = json.loads(newj, object_pairs_hook=OrderedDict) return j