def test_string_stream(self):
with warnings.catch_warnings(record=True) as warns:
events = list(self.backend.basic_parse(StringIO(b2s(JSON))))
self.assertEqual(events, JSON_EVENTS)
if self.warn_on_string_stream:
self.assertEqual(len(warns), 1)
self.assertEqual(DeprecationWarning, warns[0].category)
def test_string_stream(self):
with warning_catcher() as warns:
events = self.get_all(self.basic_parse, b2s(JSON))
self.assertEqual(events, JSON_EVENTS)
if self.warn_on_string_stream:
self.assertEqual(len(warns), 1)
self.assertEqual(DeprecationWarning, warns[0].category)
def yajl_parse(handle, buffer):
if buffer:
result = yajl.yajl_parse(handle, buffer, len(buffer))
else:
result = yajl.yajl_complete_parse(handle)
if result != YAJL_OK:
perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
error = b2s(ffi.string(perror))
yajl.yajl_free_error(handle, perror)
exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError
raise exception(error)
def yajl_parse(handle, buffer):
if buffer:
result = yajl.yajl_parse(handle, buffer, len(buffer))
else:
result = yajl.yajl_complete_parse(handle)
if result != YAJL_OK:
perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
error = b2s(ffi.string(perror))
yajl.yajl_free_error(handle, perror)
exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError
raise exception(error)
def _do_test_dump(self, method, multiple_values):
# Use python backend to ensure multiple_values works
env = dict(os.environ)
env['IJSON_BACKEND'] = 'python'
# Ensure printing works on the subprocess in Windows
# by using utf-8 on its stdout
if 'win' in sys.platform:
env = dict(os.environ)
env['PYTHONIOENCODING'] = 'utf-8'
cmd = [sys.executable, '-m', 'ijson.dump', '-m', method, '-p', '']
if multiple_values:
cmd.append('-M')
proc = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
input_data = JSON
if multiple_values:
input_data += JSON
out, err = proc.communicate(input_data)
status = proc.wait()
self.assertEqual(
0, status, "out:\n%s\nerr:%s" % (compat.b2s(out), compat.b2s(err)))
def _get_callback_data(yajl_version):
return [
# Mapping of JSON parser events to callback C types and value converters.
# Used to define the Callbacks structure and actual callback functions
# inside the parse function.
('null', 'null', C_EMPTY, lambda: None),
('boolean', 'boolean', C_INT, lambda v: bool(v)),
('integer', 'number', C_LONG if yajl_version == 1 else C_LONGLONG, lambda v: int(v)),
('double', 'number', C_DOUBLE, lambda v: v),
('number', 'number', C_STR, lambda v, l: common.integer_or_decimal(b2s(string_at(v, l)))),
('string', 'string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
('start_map', 'start_map', C_EMPTY, lambda: None),
('map_key', 'map_key', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
('end_map', 'end_map', C_EMPTY, lambda: None),
('start_array', 'start_array', C_EMPTY, lambda: None),
('end_array', 'end_array', C_EMPTY, lambda: None),
]
def __next__(self):
while True:
match = NONWS.search(self.buffer, self.pos)
if match:
self.pos = match.start()
char = self.buffer[self.pos]
if 'a' <= char <= 'z' or '0' <= char <= '9' or char == '-':
return self.lexem()
elif char == '"':
return self.stringlexem()
else:
self.pos += 1
return char
self.buffer = b2s(self.f.read(BUFSIZE))
self.pos = 0
if not len(self.buffer):
raise StopIteration
def lexem(self):
current = self.pos
while True:
match = LEXTERM.search(self.buffer, current)
if match:
current = match.start()
break
else:
current = len(self.buffer)
self.buffer += b2s(self.f.read(BUFSIZE))
if len(self.buffer) == current:
break
result = self.buffer[self.pos:current]
self.pos = current
if self.pos > BUFSIZE:
self.buffer = self.buffer[self.pos:]
self.pos = 0
return result
def stringlexem(self):
start = self.pos + 1
while True:
try:
end = self.buffer.index('"', start)
escpos = end - 1
while self.buffer[escpos] == '\\':
escpos -= 1
if (end - escpos) % 2 == 0:
start = end + 1
else:
result = self.buffer[self.pos:end + 1]
self.pos = end + 1
return result
except ValueError:
old_len = len(self.buffer)
self.buffer += b2s(self.f.read(BUFSIZE))
if len(self.buffer) == old_len:
raise common.IncompleteJSONError()
def map_key(key, length):
return b2s(ffi.string(key, maxlen=length))
def number(val, length):
return common.number(b2s(ffi.string(val, maxlen=length)))
C_INT = CFUNCTYPE(c_int, c_void_p, c_int)
C_LONG = CFUNCTYPE(c_int, c_void_p, c_long)
C_DOUBLE = CFUNCTYPE(c_int, c_void_p, c_double)
C_STR = CFUNCTYPE(c_int, c_void_p, POINTER(c_ubyte), c_uint)
_callback_data = [
# Mapping of JSON parser events to callback C types and value converters.
# Used to define the Callbacks structure and actual callback functions
# inside the parse function.
('null', C_EMPTY, lambda: None),
('boolean', C_INT, lambda v: bool(v)),
# "integer" and "double" aren't actually yielded by yajl since "number"
# takes precedence if defined
('integer', C_LONG, lambda *_args: None),
('double', C_DOUBLE, lambda *_args: None),
('number', C_STR, lambda v, l: common.number(b2s(string_at(v, l)))),
('string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
('start_map', C_EMPTY, lambda: None),
('map_key', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
('end_map', C_EMPTY, lambda: None),
('start_array', C_EMPTY, lambda: None),
('end_array', C_EMPTY, lambda: None),
]
class Callbacks(Structure):
_fields_ = [(name, type) for name, type, func in _callback_data]
class Config(Structure):
_fields_ = [("allowComments", c_uint), ("checkUTF8", c_uint)]
try:
return int(value)
except ValueError:
return Decimal(value)
_callback_data = [
# Mapping of JSON parser events to callback C types and value converters.
# Used to define the Callbacks structure and actual callback functions
# inside the parse function.
('null', C_EMPTY, lambda: None),
('boolean', C_INT, lambda v: bool(v)),
# "integer" and "double" aren't actually yielded by yajl since "number"
# takes precedence if defined
('integer', C_LONG, lambda v, l: int(string_at(v, l))),
('double', C_DOUBLE, lambda v, l: float(string_at(v, l))),
('number', C_STR, lambda v, l: number(b2s(string_at(v, l)))),
('string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
('start_map', C_EMPTY, lambda: None),
('map_key', C_STR, lambda v, l: b2s(string_at(v, l))),
('end_map', C_EMPTY, lambda: None),
('start_array', C_EMPTY, lambda: None),
('end_array', C_EMPTY, lambda: None),
]
class Callbacks(Structure):
_fields_ = [(name, type) for name, type, func in _callback_data]
class Config(Structure):
_fields_ = [
("allowComments", c_uint),
("checkUTF8", c_uint)
def map_key(key, length):
return b2s(ffi.string(key, maxlen=length))
return int(value)
except ValueError:
return Decimal(value)
_callback_data = [
# Mapping of JSON parser events to callback C types and value converters.
# Used to define the Callbacks structure and actual callback functions
# inside the parse function.
("null", C_EMPTY, lambda: None),
("boolean", C_INT, lambda v: bool(v)),
# "integer" and "double" aren't actually yielded by yajl since "number"
# takes precedence if defined
("integer", C_LONG, lambda v, l: int(string_at(v, l))),
("double", C_DOUBLE, lambda v, l: float(string_at(v, l))),
("number", C_STR, lambda v, l: number(b2s(string_at(v, l)))),
("string", C_STR, lambda v, l: string_at(v, l).decode("utf-8")),
("start_map", C_EMPTY, lambda: None),
("map_key", C_STR, lambda v, l: b2s(string_at(v, l))),
("end_map", C_EMPTY, lambda: None),
("start_array", C_EMPTY, lambda: None),
("end_array", C_EMPTY, lambda: None),
]
class Callbacks(Structure):
_fields_ = [(name, type) for name, type, func in _callback_data]
class Config(Structure):
_fields_ = [("allowComments", c_uint), ("checkUTF8", c_uint)]
C_EMPTY = CFUNCTYPE(c_int, c_void_p)
C_INT = CFUNCTYPE(c_int, c_void_p, c_int)
C_LONG = CFUNCTYPE(c_int, c_void_p, c_long)
C_DOUBLE = CFUNCTYPE(c_int, c_void_p, c_double)
C_STR = CFUNCTYPE(c_int, c_void_p, POINTER(c_ubyte), c_uint)
_callback_data = [
# Mapping of JSON parser events to callback C types and value converters.
# Used to define the Callbacks structure and actual callback functions
# inside the parse function.
('null', 'null', C_EMPTY, lambda: None),
('boolean', 'boolean', C_INT, lambda v: bool(v)),
('integer', 'number', C_LONG, lambda v: v),
('double', 'number', C_DOUBLE, lambda v: v),
('number', 'number', C_STR,
lambda v, l: common.integer_or_decimal(b2s(string_at(v, l)))),
('string', 'string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
('start_map', 'start_map', C_EMPTY, lambda: None),
('map_key', 'map_key', C_STR,
lambda v, l: string_at(v, l).decode('utf-8')),
('end_map', 'end_map', C_EMPTY, lambda: None),
('start_array', 'start_array', C_EMPTY, lambda: None),
('end_array', 'end_array', C_EMPTY, lambda: None),
]
class Callbacks(Structure):
_fields_ = [(name, type) for name, _, type, _ in _callback_data]
YAJL_OK = 0
def number(val, length):
return common.integer_or_decimal(b2s(ffi.string(val, maxlen=length)))
def __init__(self, raw_value):
super(SingleReadFileStr, self).__init__(b2s(raw_value))
def number(val, length):
return common.number(b2s(ffi.string(val, maxlen=length)))
def _assert_str(self, expected_results, routine, *args, **kwargs):
with warning_catcher() as warns:
results = list(routine(compat.b2s(JSON), *args, **kwargs))
self.assertEqual(expected_results, results)
if self.warn_on_string_stream:
self.assertEqual(1, len(warns))
