def basic_parse(f, allow_comments=False, buf_size=64 * 1024, multiple_values=False): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - buf_size: a size of an input buffer - multiple_values: allows the parser to parse multiple JSON objects ''' events = [] def callback(event, func_type, func): def c_callback(context, *args): events.append((event, func(*args))) return 1 return func_type(c_callback) callbacks = Callbacks(*[callback(*data) for data in _callback_data]) handle = yajl.yajl_alloc(byref(callbacks), None, None) if allow_comments: yajl.yajl_config(handle, YAJL_ALLOW_COMMENTS, 1) if multiple_values: yajl.yajl_config(handle, YAJL_MULTIPLE_VALUES, 1) try: while True: buffer = f.read(buf_size) if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_complete_parse(handle) if result == YAJL_ERROR: perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer)) error = cast(perror, c_char_p).value yajl.yajl_free_error(handle, perror) raise common.JSONError(error) if not buffer and not events: if result == YAJL_INSUFFICIENT_DATA: raise common.IncompleteJSONError() break for event in events: yield event events = [] finally: yajl.yajl_free(handle)
def basic_parse(f, allow_comments=False, check_utf8=False, buf_size=64 * 1024): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - check_utf8: if True, parser will cause an error if input is invalid utf-8 - buf_size: a size of an input buffer ''' f = compat.bytes_reader(f) events = [] def callback(event, func_type, func): def c_callback(context, *args): events.append((event, func(*args))) return 1 return func_type(c_callback) callbacks = Callbacks(*[callback(*data) for data in _callback_data]) config = Config(allow_comments, check_utf8) handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None) try: while True: buffer = f.read(buf_size) if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_parse_complete(handle) if result == YAJL_ERROR: perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer)) error = cast(perror, c_char_p).value yajl.yajl_free_error(handle, perror) exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError raise common.JSONError(error) if not buffer and not events: if result == YAJL_INSUFFICIENT_DATA: raise common.IncompleteJSONError('Incomplete JSON data') break for event in events: yield event events = [] finally: yajl.yajl_free(handle)
def basic_parse(file): ''' Iterator yielding unprefixed events. Parameters: - file: a readable file-like object with JSON input ''' lexer = iter(Lexer(file)) for value in parse_value(lexer): yield value try: next(lexer) except StopIteration: pass else: raise common.JSONError('Additional data')
def basic_parse_basecoro(target, allow_comments=False, check_utf8=False): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - check_utf8: if True, parser will cause an error if input is invalid utf-8 - buf_size: a size of an input buffer ''' send = target.send def callback(event, func_type, func): def c_callback(context, *args): send((event, func(*args))) return 1 return func_type(c_callback) callbacks = Callbacks(*[callback(*data) for data in _callback_data]) config = Config(allow_comments, check_utf8) handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None) try: while True: try: buffer = (yield) except GeneratorExit: buffer = b'' if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_parse_complete(handle) if result == YAJL_ERROR: perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer)) error = cast(perror, c_char_p).value yajl.yajl_free_error(handle, perror) raise common.JSONError(error.decode('utf-8')) elif not buffer: if result == YAJL_INSUFFICIENT_DATA: raise common.IncompleteJSONError('Incomplete JSON data') break finally: yajl.yajl_free(handle)
def basic_parse(file, buf_size=BUFSIZE, do_translate=True): ''' Iterator yielding unprefixed events. Parameters: - file: a readable file-like object with JSON input ''' global DOTRANSLATE DOTRANSLATE = do_translate lexer = iter(Lexer(file, buf_size)) for value in parse_value(lexer): yield value try: next(lexer) except StopIteration: pass else: raise common.JSONError('Additional data')
def basic_parse(file, buf_size=BUFSIZE, multiple_values=False): ''' Iterator yielding unprefixed events. Parameters: - file: a readable file-like object with JSON input ''' lexer = iter(Lexer(file, buf_size)) symbol = None pos = 0 while True: for value in parse_value(lexer, symbol, pos): yield value try: pos, symbol = next(lexer) except StopIteration: break else: if not multiple_values: raise common.JSONError('Additional data')
def basic_parse_basecoro(target, allow_comments=False, multiple_values=False, use_float=False): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - check_utf8: if True, parser will cause an error if input is invalid utf-8 - buf_size: a size of an input buffer ''' if multiple_values: raise ValueError("yajl backend doesn't support multiple_values") callbacks = _yajl2_ctypes_common.make_callbaks(target.send, use_float, 1) config = Config(allow_comments, True) handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None) try: while True: try: buffer = (yield) except GeneratorExit: buffer = b'' if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_parse_complete(handle) if result == _yajl2_ctypes_common.YAJL_ERROR: error = _yajl2_ctypes_common.yajl_get_error( yajl, handle, buffer) raise common.JSONError(error) elif not buffer: if result == _yajl2_ctypes_common.YAJL_INSUFFICIENT_DATA: raise common.IncompleteJSONError('Incomplete JSON data') break finally: yajl.yajl_free(handle)
def parse_value(target, multivalue, use_float): """ Parses results coming out of the Lexer into ijson events, which are sent to `target`. A stack keeps track of the type of object being parsed at the time (a value, and object or array -- the last two being values themselves). A special EOF result coming from the Lexer indicates that no more content is expected. This is used to check for incomplete content and raise the appropriate exception, which wouldn't be possible if the Lexer simply closed this co-routine (either explicitly via .close(), or implicitly by itself finishing and decreasing the only reference to the co-routine) since that causes a GeneratorExit exception that cannot be replaced with a custom one. """ state_stack = [_PARSE_VALUE] pop = state_stack.pop push = state_stack.append send = target.send prev_pos, prev_symbol = None, None to_number = common.integer_or_float if use_float else common.integer_or_decimal while True: if prev_pos is None: pos, symbol = (yield) if (pos, symbol) == EOF: if state_stack: raise common.IncompleteJSONError('Incomplete JSON content') break else: pos, symbol = prev_pos, prev_symbol prev_pos, prev_symbol = None, None try: state = state_stack[-1] except IndexError: if multivalue: state = _PARSE_VALUE push(state) else: raise common.JSONError('Additional data found') assert state_stack if state == _PARSE_VALUE: # Simple, common cases if symbol == 'null': send(('null', None)) pop() elif symbol == 'true': send(('boolean', True)) pop() elif symbol == 'false': send(('boolean', False)) pop() elif symbol[0] == '"': send(('string', parse_string(symbol))) pop() # Array start elif symbol == '[': send(('start_array', None)) pos, symbol = (yield) if (pos, symbol) == EOF: raise common.IncompleteJSONError('Incomplete JSON content') if symbol == ']': send(('end_array', None)) pop() else: prev_pos, prev_symbol = pos, symbol push(_PARSE_ARRAY_ELEMENT_END) push(_PARSE_VALUE) # Object start elif symbol == '{': send(('start_map', None)) pos, symbol = (yield) if (pos, symbol) == EOF: raise common.IncompleteJSONError('Incomplete JSON content') if symbol == '}': send(('end_map', None)) pop() else: prev_pos, prev_symbol = pos, symbol push(_PARSE_OBJECT_KEY) # A number else: # JSON numbers can't contain leading zeros if ((len(symbol) > 1 and symbol[0] == '0' and symbol[1] not in ('e', 'E', '.')) or (len(symbol) > 2 and symbol[0:2] == '-0' and symbol[2] not in ('e', 'E', '.'))): raise common.JSONError('Invalid JSON number: %s' % (symbol, )) # Fractions need a leading digit and must be followed by a digit if symbol[0] == '.' or symbol[-1] == '.': raise common.JSONError('Invalid JSON number: %s' % (symbol, )) try: number = to_number(symbol) if number == inf: raise common.JSONError("float overflow: %s" % (symbol, )) except: raise UnexpectedSymbol(symbol, pos) else: send(('number', number)) pop() elif state == _PARSE_OBJECT_KEY: if symbol[0] != '"': raise UnexpectedSymbol(symbol, pos) send(('map_key', parse_string(symbol))) pos, symbol = (yield) if (pos, symbol) == EOF: raise common.IncompleteJSONError('Incomplete JSON content') if symbol != ':': raise UnexpectedSymbol(symbol, pos) state_stack[-1] = _PARSE_OBJECT_END push(_PARSE_VALUE) elif state == _PARSE_OBJECT_END: if symbol == ',': state_stack[-1] = _PARSE_OBJECT_KEY elif symbol != '}': raise UnexpectedSymbol(symbol, pos) else: send(('end_map', None)) pop() pop() elif state == _PARSE_ARRAY_ELEMENT_END: if symbol == ',': state_stack[-1] = _PARSE_ARRAY_ELEMENT_END push(_PARSE_VALUE) elif symbol != ']': raise UnexpectedSymbol(symbol, pos) else: send(('end_array', None)) pop() pop()