def basic_parse(f,
allow_comments=False,
buf_size=64 * 1024,
multiple_values=False):
'''
Iterator yielding unprefixed events.
Parameters:
- f: a readable file-like object with JSON input
- allow_comments: tells parser to allow comments in JSON input
- buf_size: a size of an input buffer
- multiple_values: allows the parser to parse multiple JSON objects
'''
events = []
def callback(event, func_type, func):
def c_callback(context, *args):
events.append((event, func(*args)))
return 1
return func_type(c_callback)
callbacks = Callbacks(*[callback(*data) for data in _callback_data])
handle = yajl.yajl_alloc(byref(callbacks), None, None)
if allow_comments:
yajl.yajl_config(handle, YAJL_ALLOW_COMMENTS, 1)
if multiple_values:
yajl.yajl_config(handle, YAJL_MULTIPLE_VALUES, 1)
try:
while True:
buffer = f.read(buf_size)
if buffer:
result = yajl.yajl_parse(handle, buffer, len(buffer))
else:
result = yajl.yajl_complete_parse(handle)
if result == YAJL_ERROR:
perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
error = cast(perror, c_char_p).value
yajl.yajl_free_error(handle, perror)
raise common.JSONError(error)
if not buffer and not events:
if result == YAJL_INSUFFICIENT_DATA:
raise common.IncompleteJSONError()
break
for event in events:
yield event
events = []
finally:
yajl.yajl_free(handle)
def basic_parse(f, allow_comments=False, check_utf8=False, buf_size=64 * 1024):
'''
Iterator yielding unprefixed events.
Parameters:
- f: a readable file-like object with JSON input
- allow_comments: tells parser to allow comments in JSON input
- check_utf8: if True, parser will cause an error if input is invalid utf-8
- buf_size: a size of an input buffer
'''
f = compat.bytes_reader(f)
events = []
def callback(event, func_type, func):
def c_callback(context, *args):
events.append((event, func(*args)))
return 1
return func_type(c_callback)
callbacks = Callbacks(*[callback(*data) for data in _callback_data])
config = Config(allow_comments, check_utf8)
handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None)
try:
while True:
buffer = f.read(buf_size)
if buffer:
result = yajl.yajl_parse(handle, buffer, len(buffer))
else:
result = yajl.yajl_parse_complete(handle)
if result == YAJL_ERROR:
perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
error = cast(perror, c_char_p).value
yajl.yajl_free_error(handle, perror)
exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError
raise common.JSONError(error)
if not buffer and not events:
if result == YAJL_INSUFFICIENT_DATA:
raise common.IncompleteJSONError('Incomplete JSON data')
break
for event in events:
yield event
events = []
finally:
yajl.yajl_free(handle)
def basic_parse(file):
'''
Iterator yielding unprefixed events.
Parameters:
- file: a readable file-like object with JSON input
'''
lexer = iter(Lexer(file))
for value in parse_value(lexer):
yield value
try:
next(lexer)
except StopIteration:
pass
else:
raise common.JSONError('Additional data')
def basic_parse_basecoro(target, allow_comments=False, check_utf8=False):
'''
Iterator yielding unprefixed events.
Parameters:
- f: a readable file-like object with JSON input
- allow_comments: tells parser to allow comments in JSON input
- check_utf8: if True, parser will cause an error if input is invalid utf-8
- buf_size: a size of an input buffer
'''
send = target.send
def callback(event, func_type, func):
def c_callback(context, *args):
send((event, func(*args)))
return 1
return func_type(c_callback)
callbacks = Callbacks(*[callback(*data) for data in _callback_data])
config = Config(allow_comments, check_utf8)
handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None)
try:
while True:
try:
buffer = (yield)
except GeneratorExit:
buffer = b''
if buffer:
result = yajl.yajl_parse(handle, buffer, len(buffer))
else:
result = yajl.yajl_parse_complete(handle)
if result == YAJL_ERROR:
perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
error = cast(perror, c_char_p).value
yajl.yajl_free_error(handle, perror)
raise common.JSONError(error.decode('utf-8'))
elif not buffer:
if result == YAJL_INSUFFICIENT_DATA:
raise common.IncompleteJSONError('Incomplete JSON data')
break
finally:
yajl.yajl_free(handle)
def basic_parse(file, buf_size=BUFSIZE, do_translate=True):
'''
Iterator yielding unprefixed events.
Parameters:
- file: a readable file-like object with JSON input
'''
global DOTRANSLATE
DOTRANSLATE = do_translate
lexer = iter(Lexer(file, buf_size))
for value in parse_value(lexer):
yield value
try:
next(lexer)
except StopIteration:
pass
else:
raise common.JSONError('Additional data')
def basic_parse(file, buf_size=BUFSIZE, multiple_values=False):
'''
Iterator yielding unprefixed events.
Parameters:
- file: a readable file-like object with JSON input
'''
lexer = iter(Lexer(file, buf_size))
symbol = None
pos = 0
while True:
for value in parse_value(lexer, symbol, pos):
yield value
try:
pos, symbol = next(lexer)
except StopIteration:
break
else:
if not multiple_values:
raise common.JSONError('Additional data')
def basic_parse_basecoro(target,
allow_comments=False,
multiple_values=False,
use_float=False):
'''
Iterator yielding unprefixed events.
Parameters:
- f: a readable file-like object with JSON input
- allow_comments: tells parser to allow comments in JSON input
- check_utf8: if True, parser will cause an error if input is invalid utf-8
- buf_size: a size of an input buffer
'''
if multiple_values:
raise ValueError("yajl backend doesn't support multiple_values")
callbacks = _yajl2_ctypes_common.make_callbaks(target.send, use_float, 1)
config = Config(allow_comments, True)
handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None)
try:
while True:
try:
buffer = (yield)
except GeneratorExit:
buffer = b''
if buffer:
result = yajl.yajl_parse(handle, buffer, len(buffer))
else:
result = yajl.yajl_parse_complete(handle)
if result == _yajl2_ctypes_common.YAJL_ERROR:
error = _yajl2_ctypes_common.yajl_get_error(
yajl, handle, buffer)
raise common.JSONError(error)
elif not buffer:
if result == _yajl2_ctypes_common.YAJL_INSUFFICIENT_DATA:
raise common.IncompleteJSONError('Incomplete JSON data')
break
finally:
yajl.yajl_free(handle)
def parse_value(target, multivalue, use_float):
"""
Parses results coming out of the Lexer into ijson events, which are sent to
`target`. A stack keeps track of the type of object being parsed at the time
(a value, and object or array -- the last two being values themselves).
A special EOF result coming from the Lexer indicates that no more content is
expected. This is used to check for incomplete content and raise the
appropriate exception, which wouldn't be possible if the Lexer simply closed
this co-routine (either explicitly via .close(), or implicitly by itself
finishing and decreasing the only reference to the co-routine) since that
causes a GeneratorExit exception that cannot be replaced with a custom one.
"""
state_stack = [_PARSE_VALUE]
pop = state_stack.pop
push = state_stack.append
send = target.send
prev_pos, prev_symbol = None, None
to_number = common.integer_or_float if use_float else common.integer_or_decimal
while True:
if prev_pos is None:
pos, symbol = (yield)
if (pos, symbol) == EOF:
if state_stack:
raise common.IncompleteJSONError('Incomplete JSON content')
break
else:
pos, symbol = prev_pos, prev_symbol
prev_pos, prev_symbol = None, None
try:
state = state_stack[-1]
except IndexError:
if multivalue:
state = _PARSE_VALUE
push(state)
else:
raise common.JSONError('Additional data found')
assert state_stack
if state == _PARSE_VALUE:
# Simple, common cases
if symbol == 'null':
send(('null', None))
pop()
elif symbol == 'true':
send(('boolean', True))
pop()
elif symbol == 'false':
send(('boolean', False))
pop()
elif symbol[0] == '"':
send(('string', parse_string(symbol)))
pop()
# Array start
elif symbol == '[':
send(('start_array', None))
pos, symbol = (yield)
if (pos, symbol) == EOF:
raise common.IncompleteJSONError('Incomplete JSON content')
if symbol == ']':
send(('end_array', None))
pop()
else:
prev_pos, prev_symbol = pos, symbol
push(_PARSE_ARRAY_ELEMENT_END)
push(_PARSE_VALUE)
# Object start
elif symbol == '{':
send(('start_map', None))
pos, symbol = (yield)
if (pos, symbol) == EOF:
raise common.IncompleteJSONError('Incomplete JSON content')
if symbol == '}':
send(('end_map', None))
pop()
else:
prev_pos, prev_symbol = pos, symbol
push(_PARSE_OBJECT_KEY)
# A number
else:
# JSON numbers can't contain leading zeros
if ((len(symbol) > 1 and symbol[0] == '0'
and symbol[1] not in ('e', 'E', '.'))
or (len(symbol) > 2 and symbol[0:2] == '-0'
and symbol[2] not in ('e', 'E', '.'))):
raise common.JSONError('Invalid JSON number: %s' %
(symbol, ))
# Fractions need a leading digit and must be followed by a digit
if symbol[0] == '.' or symbol[-1] == '.':
raise common.JSONError('Invalid JSON number: %s' %
(symbol, ))
try:
number = to_number(symbol)
if number == inf:
raise common.JSONError("float overflow: %s" %
(symbol, ))
except:
raise UnexpectedSymbol(symbol, pos)
else:
send(('number', number))
pop()
elif state == _PARSE_OBJECT_KEY:
if symbol[0] != '"':
raise UnexpectedSymbol(symbol, pos)
send(('map_key', parse_string(symbol)))
pos, symbol = (yield)
if (pos, symbol) == EOF:
raise common.IncompleteJSONError('Incomplete JSON content')
if symbol != ':':
raise UnexpectedSymbol(symbol, pos)
state_stack[-1] = _PARSE_OBJECT_END
push(_PARSE_VALUE)
elif state == _PARSE_OBJECT_END:
if symbol == ',':
state_stack[-1] = _PARSE_OBJECT_KEY
elif symbol != '}':
raise UnexpectedSymbol(symbol, pos)
else:
send(('end_map', None))
pop()
pop()
elif state == _PARSE_ARRAY_ELEMENT_END:
if symbol == ',':
state_stack[-1] = _PARSE_ARRAY_ELEMENT_END
push(_PARSE_VALUE)
elif symbol != ']':
raise UnexpectedSymbol(symbol, pos)
else:
send(('end_array', None))
pop()
pop()
