Exemple #1
0
 def test_string_stream(self):
     with warnings.catch_warnings(record=True) as warns:
         events = list(self.backend.basic_parse(StringIO(b2s(JSON))))
         self.assertEqual(events, JSON_EVENTS)
     if self.warn_on_string_stream:
         self.assertEqual(len(warns), 1)
         self.assertEqual(DeprecationWarning, warns[0].category)
Exemple #2
0
 def test_string_stream(self):
     with warning_catcher() as warns:
         events = self.get_all(self.basic_parse, b2s(JSON))
         self.assertEqual(events, JSON_EVENTS)
     if self.warn_on_string_stream:
         self.assertEqual(len(warns), 1)
         self.assertEqual(DeprecationWarning, warns[0].category)
Exemple #3
0
def yajl_parse(handle, buffer):
    if buffer:
        result = yajl.yajl_parse(handle, buffer, len(buffer))
    else:
        result = yajl.yajl_complete_parse(handle)

    if result != YAJL_OK:
        perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
        error = b2s(ffi.string(perror))
        yajl.yajl_free_error(handle, perror)
        exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError
        raise exception(error)
def yajl_parse(handle, buffer):
    if buffer:
        result = yajl.yajl_parse(handle, buffer, len(buffer))
    else:
        result = yajl.yajl_complete_parse(handle)

    if result != YAJL_OK:
        perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
        error = b2s(ffi.string(perror))
        yajl.yajl_free_error(handle, perror)
        exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError
        raise exception(error)
Exemple #5
0
 def _do_test_dump(self, method, multiple_values):
     # Use python backend to ensure multiple_values works
     env = dict(os.environ)
     env['IJSON_BACKEND'] = 'python'
     # Ensure printing works on the subprocess in Windows
     # by using utf-8 on its stdout
     if 'win' in sys.platform:
         env = dict(os.environ)
         env['PYTHONIOENCODING'] = 'utf-8'
     cmd = [sys.executable, '-m', 'ijson.dump', '-m', method, '-p', '']
     if multiple_values:
         cmd.append('-M')
     proc = subprocess.Popen(cmd,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             env=env)
     input_data = JSON
     if multiple_values:
         input_data += JSON
     out, err = proc.communicate(input_data)
     status = proc.wait()
     self.assertEqual(
         0, status, "out:\n%s\nerr:%s" % (compat.b2s(out), compat.b2s(err)))
def _get_callback_data(yajl_version):
    return  [
        # Mapping of JSON parser events to callback C types and value converters.
        # Used to define the Callbacks structure and actual callback functions
        # inside the parse function.
        ('null', 'null', C_EMPTY, lambda: None),
        ('boolean', 'boolean', C_INT, lambda v: bool(v)),
        ('integer', 'number', C_LONG if yajl_version == 1 else C_LONGLONG, lambda v: int(v)),
        ('double', 'number', C_DOUBLE, lambda v: v),
        ('number', 'number', C_STR, lambda v, l: common.integer_or_decimal(b2s(string_at(v, l)))),
        ('string', 'string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
        ('start_map', 'start_map', C_EMPTY, lambda: None),
        ('map_key', 'map_key', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
        ('end_map', 'end_map', C_EMPTY, lambda: None),
        ('start_array', 'start_array', C_EMPTY, lambda: None),
        ('end_array', 'end_array', C_EMPTY, lambda: None),
    ]
Exemple #7
0
 def __next__(self):
     while True:
         match = NONWS.search(self.buffer, self.pos)
         if match:
             self.pos = match.start()
             char = self.buffer[self.pos]
             if 'a' <= char <= 'z' or '0' <= char <= '9' or char == '-':
                 return self.lexem()
             elif char == '"':
                 return self.stringlexem()
             else:
                 self.pos += 1
                 return char
         self.buffer = b2s(self.f.read(BUFSIZE))
         self.pos = 0
         if not len(self.buffer):
             raise StopIteration
Exemple #8
0
 def lexem(self):
     current = self.pos
     while True:
         match = LEXTERM.search(self.buffer, current)
         if match:
             current = match.start()
             break
         else:
             current = len(self.buffer)
             self.buffer += b2s(self.f.read(BUFSIZE))
             if len(self.buffer) == current:
                 break
     result = self.buffer[self.pos:current]
     self.pos = current
     if self.pos > BUFSIZE:
         self.buffer = self.buffer[self.pos:]
         self.pos = 0
     return result
Exemple #9
0
 def stringlexem(self):
     start = self.pos + 1
     while True:
         try:
             end = self.buffer.index('"', start)
             escpos = end - 1
             while self.buffer[escpos] == '\\':
                 escpos -= 1
             if (end - escpos) % 2 == 0:
                 start = end + 1
             else:
                 result = self.buffer[self.pos:end + 1]
                 self.pos = end + 1
                 return result
         except ValueError:
             old_len = len(self.buffer)
             self.buffer += b2s(self.f.read(BUFSIZE))
             if len(self.buffer) == old_len:
                 raise common.IncompleteJSONError()
Exemple #10
0
def map_key(key, length):
    return b2s(ffi.string(key, maxlen=length))
Exemple #11
0
def number(val, length):
    return common.number(b2s(ffi.string(val, maxlen=length)))
Exemple #12
0
C_INT = CFUNCTYPE(c_int, c_void_p, c_int)
C_LONG = CFUNCTYPE(c_int, c_void_p, c_long)
C_DOUBLE = CFUNCTYPE(c_int, c_void_p, c_double)
C_STR = CFUNCTYPE(c_int, c_void_p, POINTER(c_ubyte), c_uint)

_callback_data = [
    # Mapping of JSON parser events to callback C types and value converters.
    # Used to define the Callbacks structure and actual callback functions
    # inside the parse function.
    ('null', C_EMPTY, lambda: None),
    ('boolean', C_INT, lambda v: bool(v)),
    # "integer" and "double" aren't actually yielded by yajl since "number"
    # takes precedence if defined
    ('integer', C_LONG, lambda *_args: None),
    ('double', C_DOUBLE, lambda *_args: None),
    ('number', C_STR, lambda v, l: common.number(b2s(string_at(v, l)))),
    ('string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
    ('start_map', C_EMPTY, lambda: None),
    ('map_key', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
    ('end_map', C_EMPTY, lambda: None),
    ('start_array', C_EMPTY, lambda: None),
    ('end_array', C_EMPTY, lambda: None),
]


class Callbacks(Structure):
    _fields_ = [(name, type) for name, type, func in _callback_data]


class Config(Structure):
    _fields_ = [("allowComments", c_uint), ("checkUTF8", c_uint)]
Exemple #13
0
    try:
        return int(value)
    except ValueError:
        return Decimal(value)

_callback_data = [
    # Mapping of JSON parser events to callback C types and value converters.
    # Used to define the Callbacks structure and actual callback functions
    # inside the parse function.
    ('null', C_EMPTY, lambda: None),
    ('boolean', C_INT, lambda v: bool(v)),
    # "integer" and "double" aren't actually yielded by yajl since "number"
    # takes precedence if defined
    ('integer', C_LONG, lambda v, l: int(string_at(v, l))),
    ('double', C_DOUBLE, lambda v, l: float(string_at(v, l))),
    ('number', C_STR, lambda v, l: number(b2s(string_at(v, l)))),
    ('string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
    ('start_map', C_EMPTY, lambda: None),
    ('map_key', C_STR, lambda v, l: b2s(string_at(v, l))),
    ('end_map', C_EMPTY, lambda: None),
    ('start_array', C_EMPTY, lambda: None),
    ('end_array', C_EMPTY, lambda: None),
]

class Callbacks(Structure):
    _fields_ = [(name, type) for name, type, func in _callback_data]

class Config(Structure):
    _fields_ = [
        ("allowComments", c_uint),
        ("checkUTF8", c_uint)
Exemple #14
0
def map_key(key, length):
    return b2s(ffi.string(key, maxlen=length))
Exemple #15
0
        return int(value)
    except ValueError:
        return Decimal(value)


_callback_data = [
    # Mapping of JSON parser events to callback C types and value converters.
    # Used to define the Callbacks structure and actual callback functions
    # inside the parse function.
    ("null", C_EMPTY, lambda: None),
    ("boolean", C_INT, lambda v: bool(v)),
    # "integer" and "double" aren't actually yielded by yajl since "number"
    # takes precedence if defined
    ("integer", C_LONG, lambda v, l: int(string_at(v, l))),
    ("double", C_DOUBLE, lambda v, l: float(string_at(v, l))),
    ("number", C_STR, lambda v, l: number(b2s(string_at(v, l)))),
    ("string", C_STR, lambda v, l: string_at(v, l).decode("utf-8")),
    ("start_map", C_EMPTY, lambda: None),
    ("map_key", C_STR, lambda v, l: b2s(string_at(v, l))),
    ("end_map", C_EMPTY, lambda: None),
    ("start_array", C_EMPTY, lambda: None),
    ("end_array", C_EMPTY, lambda: None),
]


class Callbacks(Structure):
    _fields_ = [(name, type) for name, type, func in _callback_data]


class Config(Structure):
    _fields_ = [("allowComments", c_uint), ("checkUTF8", c_uint)]
Exemple #16
0
C_EMPTY = CFUNCTYPE(c_int, c_void_p)
C_INT = CFUNCTYPE(c_int, c_void_p, c_int)
C_LONG = CFUNCTYPE(c_int, c_void_p, c_long)
C_DOUBLE = CFUNCTYPE(c_int, c_void_p, c_double)
C_STR = CFUNCTYPE(c_int, c_void_p, POINTER(c_ubyte), c_uint)

_callback_data = [
    # Mapping of JSON parser events to callback C types and value converters.
    # Used to define the Callbacks structure and actual callback functions
    # inside the parse function.
    ('null', 'null', C_EMPTY, lambda: None),
    ('boolean', 'boolean', C_INT, lambda v: bool(v)),
    ('integer', 'number', C_LONG, lambda v: v),
    ('double', 'number', C_DOUBLE, lambda v: v),
    ('number', 'number', C_STR,
     lambda v, l: common.integer_or_decimal(b2s(string_at(v, l)))),
    ('string', 'string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
    ('start_map', 'start_map', C_EMPTY, lambda: None),
    ('map_key', 'map_key', C_STR,
     lambda v, l: string_at(v, l).decode('utf-8')),
    ('end_map', 'end_map', C_EMPTY, lambda: None),
    ('start_array', 'start_array', C_EMPTY, lambda: None),
    ('end_array', 'end_array', C_EMPTY, lambda: None),
]


class Callbacks(Structure):
    _fields_ = [(name, type) for name, _, type, _ in _callback_data]


YAJL_OK = 0
Exemple #17
0
def number(val, length):
    return common.integer_or_decimal(b2s(ffi.string(val, maxlen=length)))
Exemple #18
0
 def __init__(self, raw_value):
     super(SingleReadFileStr, self).__init__(b2s(raw_value))
Exemple #19
0
def number(val, length):
    return common.number(b2s(ffi.string(val, maxlen=length)))
Exemple #20
0
 def _assert_str(self, expected_results, routine, *args, **kwargs):
     with warning_catcher() as warns:
         results = list(routine(compat.b2s(JSON), *args, **kwargs))
     self.assertEqual(expected_results, results)
     if self.warn_on_string_stream:
         self.assertEqual(1, len(warns))