def items(file, prefix, map_type=None, **kwargs): ''' Backend-specific wrapper for ijson.common.items. ''' return common.items(parse(compat.bytes_reader(file), **kwargs), prefix, map_type=map_type)
def file_source(f, buf_size=64*1024): '''A generator that yields data from a file-like object''' f = compat.bytes_reader(f) while True: data = f.read(buf_size) yield data if not data: break
def basic_parse(f, allow_comments=False, buf_size=64 * 1024, multiple_values=False): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - buf_size: a size of an input buffer - multiple_values: allows the parser to parse multiple JSON objects ''' f = compat.bytes_reader(f) events = [] def callback(event, func_type, func): def c_callback(context, *args): try: value = func(*args) except: return 0 events.append((event, value)) return 1 return func_type(c_callback) callbacks = Callbacks(*[callback(*data) for data in _callback_data]) handle = yajl.yajl_alloc(byref(callbacks), None, None) if allow_comments: yajl.yajl_config(handle, YAJL_ALLOW_COMMENTS, 1) if multiple_values: yajl.yajl_config(handle, YAJL_MULTIPLE_VALUES, 1) try: while True: buffer = f.read(buf_size) if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_complete_parse(handle) if result != YAJL_OK: perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer)) error = cast(perror, c_char_p).value yajl.yajl_free_error(handle, perror) exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError raise exception(error.decode('utf-8')) if not buffer and not events: break for event in events: yield event events = [] finally: yajl.yajl_free(handle)
def basic_parse(f, allow_comments=False, check_utf8=False, buf_size=64 * 1024): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - check_utf8: if True, parser will cause an error if input is invalid utf-8 - buf_size: a size of an input buffer ''' f = compat.bytes_reader(f) events = [] def callback(event, func_type, func): def c_callback(context, *args): events.append((event, func(*args))) return 1 return func_type(c_callback) callbacks = Callbacks(*[callback(*data) for data in _callback_data]) config = Config(allow_comments, check_utf8) handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None) try: while True: buffer = f.read(buf_size) if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_parse_complete(handle) if result == YAJL_ERROR: perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer)) error = cast(perror, c_char_p).value yajl.yajl_free_error(handle, perror) exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError raise common.JSONError(error) if not buffer and not events: if result == YAJL_INSUFFICIENT_DATA: raise common.IncompleteJSONError('Incomplete JSON data') break for event in events: yield event events = [] finally: yajl.yajl_free(handle)
def basic_parse(f, buf_size=64*1024, **config): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - buf_size: a size of an input buffer - multiple_values: allows the parser to parse multiple JSON objects ''' # the scope objects makes sure the C objects allocated in _yajl.init # are kept alive until this function is done f = compat.bytes_reader(f) scope = Container() events = [] handle = yajl_init(scope, events, **config) try: while True: buffer = f.read(buf_size) # this calls the callbacks which will # fill the events list yajl_parse(handle, buffer) if not buffer and not events: break for event in events: yield event # clear all events, but don't replace the # the events list instance del events[:] finally: yajl.yajl_free(handle)
def parse(file, **kwargs): ''' Backend-specific wrapper for ijson.common.parse. ''' return common.parse(basic_parse(compat.bytes_reader(file), **kwargs))
def kvitems(file, prefix, map_type=None, **kwargs): f = compat.bytes_reader(file) return _yajl2.kvitems(prefix, f.read, decimal.Decimal, common.JSONError, common.IncompleteJSONError, map_type, **kwargs)
def parse(file, **kwargs): f = compat.bytes_reader(file) return _yajl2.parse(f.read, decimal.Decimal, common.JSONError, common.IncompleteJSONError, **kwargs)
def items_gen(file, prefix, map_type=None, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) prefix = _get_prefix(prefix) return _yajl2.items(f, buf_size, prefix, map_type, **kwargs)
def parse_gen(file, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) return _yajl2.parse(f, buf_size, **kwargs)
def items(file, prefix, map_type=None, **kwargs): f = compat.bytes_reader(file) buf_size = kwargs.pop('buf_size', 64 * 1024) return _yajl2.items(f, buf_size, prefix, map_type, **kwargs)
def parse(file, **kwargs): f = compat.bytes_reader(file) buf_size = kwargs.pop('buf_size', 64 * 1024) return _yajl2.parse(f, buf_size, **kwargs)