def test_preallocate(self): from __pypy__.builders import UnicodeBuilder b = UnicodeBuilder(10) b.append(u"abc") b.append(u"123") s = b.build() assert s == u"abc123"
class Neo4jFormatter(object): def __init__(self, fd): self.builder = UnicodeBuilder() self.count = 0 self.fd = fd def add(self, output): self.count += 1 self.builder.append(output) if self.count == BATCH_SIZE: self.writeBatch() self.count = 0 self.builder = UnicodeBuilder() else: self.builder.append(",") def writeBatch(self): built = self.builder.build() if built.endswith(","): built = built[:-1] self.fd.write("CREATE %s;\n" % built) def finished(self): self.writeBatch() def started(self): pass
def pypy_json_encode(value, pretty=False): """ pypy DOES NOT OPTIMIZE GENERATOR CODE WELL """ global _dealing_with_problem if pretty: return pretty_json(value) try: _buffer = UnicodeBuilder(2048) _value2json(value, _buffer) output = _buffer.build() return output except Exception as e: # THE PRETTY JSON WILL PROVIDE MORE DETAIL ABOUT THE SERIALIZATION CONCERNS from mo_logs import Log if _dealing_with_problem: Log.error("Serialization of JSON problems", e) else: Log.warning("Serialization of JSON problems", e) _dealing_with_problem = True try: return pretty_json(value) except Exception as f: Log.error("problem serializing object", f) finally: _dealing_with_problem = False
def test_append_slice(self): from __pypy__.builders import UnicodeBuilder b = UnicodeBuilder() b.append_slice(u"abcdefgh", 2, 5) raises(ValueError, b.append_slice, u"1", 2, 1) s = b.build() assert s == "cde" raises(ValueError, b.append_slice, u"abc", 1, 2)
def append(self, string): try: self._builder.append(string) except UnicodeEncodeError: ub = UnicodeBuilder() ub.append(self._builder.build()) self._builder = ub ub.append(string)
def append(self, string): if (isinstance(string, unicode) and type(self._builder) is StringBuilder): ub = UnicodeBuilder() ub.append(self._builder.build()) self._builder = ub self.append = ub.append # shortcut only self._builder.append(string)
def split_brackets(line): start = None in_quotes = False is_escaped = False num = 0 line_len = len(line) ret = [] accumulator = UnicodeBuilder() while num + 1 <= line_len: char = line[num] #print "Char %s: %s, %s" % (char, in_quotes, is_escaped) if char == "\\": if is_escaped: is_escaped = False else: is_escaped = True num += 1 # Skip the escape characters continue elif char == "'": if not is_escaped: in_quotes = not in_quotes else: is_escaped = False elif is_escaped: # Sometimes other characters are escaped. is_escaped = False if not in_quotes: if char == ",": char = "|" # replace commas with | elif char == "(": if start is None: start = num elif char == ")": ret.append(accumulator.build()[1:]) accumulator = UnicodeBuilder() start = None num += 1 if start: #print "Adding char %s" % char accumulator.append(char) return ret
def encode(self, value, pretty=False): if pretty: return pretty_json(value) try: _buffer = UnicodeBuilder(1024) _value2json(value, _buffer) output = _buffer.build() return output except Exception, e: #THE PRETTY JSON WILL PROVIDE MORE DETAIL ABOUT THE SERIALIZATION CONCERNS from .env.logs import Log try: pretty_json(value) except Exception, f: Log.error("problem serializing object", f)
def add(self, output): self.count += 1 self.builder.append(output) if self.count == BATCH_SIZE: self.writeBatch() self.count = 0 self.builder = UnicodeBuilder() else: self.builder.append(",")
def escape(s): if hasattr(s, '__html__'): return s.__html__() s = unicode(s) i = 0 delta = 0 repls = 0 # Find the total number of replacements, and their total distance. while i < len(s): c = ord(s[i]) try: d = ESCAPED_CHARS_DELTA_LEN[c] except IndexError: pass else: delta += d # Obscene performance hack, d >> 2 returns 0 for 0 or 1 for 4 and 5 # which are the only values that can be here. repls += d >> 2 i += 1 # Performance hack, can go away when PyPy's bridges are better # optimized del c del d # If there are no replacements just return the original string. if not repls: return s res = UnicodeBuilder(len(s) + delta) in_idx = 0 # While there are still replcaements in the string. while repls > 0: repls -= 1 next_escp = in_idx # Find the next escape while next_escp < len(s): if (ord(s[next_escp]) < len(ESCAPED_CHARS_DELTA_LEN) and ESCAPED_CHARS_DELTA_LEN[ord(s[next_escp])]): break next_escp += 1 # If we moved anywhere between escapes, copy that data. if next_escp > in_idx: res.append_slice(s, in_idx, next_escp) res.append(ESCAPED_CHARS_REPL[ord(s[next_escp])]) in_idx = next_escp + 1 # Another performance hack del next_escp # If there's anything left of the string, copy it over. if in_idx < len(s): res.append_slice(s, in_idx, len(s)) return Markup(res.build())
def encode(value, pretty=False): """ pypy DOES NOT OPTIMIZE GENERATOR CODE WELL """ if pretty: return pretty_json(value) try: _buffer = UnicodeBuilder(1024) _value2json(value, _buffer) output = _buffer.build() return output except Exception, e: # THE PRETTY JSON WILL PROVIDE MORE DETAIL ABOUT THE SERIALIZATION CONCERNS from .env.logs import Log Log.warning("Serialization of JSON problems", e) try: return pretty_json(value) except Exception, f: Log.error("problem serializing object", f)
def test_simple(self): from __pypy__.builders import UnicodeBuilder b = UnicodeBuilder() b.append(u"abc") b.append(u"123") b.append(u"1") s = b.build() assert s == u"abc1231" raises(ValueError, b.build) raises(ValueError, b.append, u"123")
class UnicodeStringBuilder(object): def __init__(self): self.b = UnicodeBuilder() def getValue(self): return self.b.build() def append(self, val): self.b.append(val) def write(self, val): self.b.append(val) def write_slice(self, *args, **kwargs): self.b.append_slice(*args, **kwargs)
def encode(self, o): """Return a JSON string representation of a Python data structure. >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) '{"foo": ["bar", "baz"]}' """ if self.check_circular: markers = {} else: markers = None if self.ensure_ascii: builder = StringBuilder() else: builder = UnicodeBuilder() self._encode(o, markers, builder, 0) return builder.build()
def test_encode(self): from __pypy__.builders import UnicodeBuilder b = UnicodeBuilder() raises(UnicodeDecodeError, b.append, b'\xc0')
def __init__(self, fd): self.builder = UnicodeBuilder() self.count = 0 self.fd = fd
def __init__(self): self.b = UnicodeBuilder()
def test_simple(self): from __pypy__.builders import UnicodeBuilder b = UnicodeBuilder() b.append(u"abc") b.append(u"123") b.append(u"1") s = b.build() assert s == u"abc1231" assert b.build() == s b.append(u"123") assert b.build() == s + u"123"