def test_common_number_is_deprecated(self):
with warnings.catch_warnings(record=True) as warns:
# 2.7 needs to enable the "always" filter to let warnings go through
if compat.IS_PY2:
warnings.simplefilter("always")
common.number("1")
self.assertEqual(len(warns), 1)
self.assertEqual(DeprecationWarning, warns[0].category)
def parse_value(lexer, symbol=None, pos=0):
try:
if symbol is None:
pos, symbol = next(lexer)
if symbol == 'null':
yield ('null', None)
elif symbol == 'true':
yield ('boolean', True)
elif symbol == 'false':
yield ('boolean', False)
elif symbol == '[':
for event in parse_array(lexer):
yield event
elif symbol == '{':
for event in parse_object(lexer):
yield event
elif symbol[0] == '"':
yield ('string', parse_string(symbol))
else:
try:
yield ('number', common.number(symbol))
except decimal.InvalidOperation:
raise UnexpectedSymbol(symbol, pos)
except StopIteration:
raise common.IncompleteJSONError('Incomplete JSON data')
def parse_value(lexer, symbol=None, pos=0):
try:
if symbol is None:
pos, symbol = next(lexer)
if symbol == 'null':
yield ('null', None)
elif symbol == 'true':
yield ('boolean', True)
elif symbol == 'false':
yield ('boolean', False)
elif symbol == '[':
for event in parse_array(lexer):
yield event
elif symbol == '{':
for event in parse_object(lexer):
yield event
elif symbol[0] == '"':
yield ('string', parse_string(symbol))
else:
try:
yield ('number', common.number(symbol))
except:
raise UnexpectedSymbol(symbol, pos)
except StopIteration:
raise common.IncompleteJSONError('Incomplete JSON data')
def parse_value(lexer, symbol=None, pos=0):
try:
if symbol is None:
pos, symbol = next(lexer)
if symbol == "null":
yield ("null", None)
elif symbol == "true":
yield ("boolean", True)
elif symbol == "false":
yield ("boolean", False)
elif symbol == "[":
for event in parse_array(lexer):
yield event
elif symbol == "{":
for event in parse_object(lexer):
yield event
elif symbol[0] == '"':
yield ("string", unescape(symbol[1:-1]))
else:
try:
yield ("number", common.number(symbol))
except decimal.InvalidOperation:
raise UnexpectedSymbol(symbol, pos)
except StopIteration:
raise common.IncompleteJSONError("Incomplete JSON data")
def number(val, length):
return common.number(b2s(ffi.string(val, maxlen=length)))
C_INT = CFUNCTYPE(c_int, c_void_p, c_int)
C_LONG = CFUNCTYPE(c_int, c_void_p, c_long)
C_DOUBLE = CFUNCTYPE(c_int, c_void_p, c_double)
C_STR = CFUNCTYPE(c_int, c_void_p, POINTER(c_ubyte), c_uint)
_callback_data = [
# Mapping of JSON parser events to callback C types and value converters.
# Used to define the Callbacks structure and actual callback functions
# inside the parse function.
('null', C_EMPTY, lambda: None),
('boolean', C_INT, lambda v: bool(v)),
# "integer" and "double" aren't actually yielded by yajl since "number"
# takes precedence if defined
('integer', C_LONG, lambda *_args: None),
('double', C_DOUBLE, lambda *_args: None),
('number', C_STR, lambda v, l: common.number(b2s(string_at(v, l)))),
('string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
('start_map', C_EMPTY, lambda: None),
('map_key', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
('end_map', C_EMPTY, lambda: None),
('start_array', C_EMPTY, lambda: None),
('end_array', C_EMPTY, lambda: None),
]
class Callbacks(Structure):
_fields_ = [(name, type) for name, type, func in _callback_data]
class Config(Structure):
_fields_ = [("allowComments", c_uint), ("checkUTF8", c_uint)]
def number(val, length):
return common.number(b2s(ffi.string(val, maxlen=length)))
def number(val, len):
return common.number(ffi.string(val, maxlen=len))
C_LONG = CFUNCTYPE(c_int, c_void_p, c_long)
C_DOUBLE = CFUNCTYPE(c_int, c_void_p, c_double)
C_STR = CFUNCTYPE(c_int, c_void_p, POINTER(c_ubyte), c_uint)
_callback_data = [
# Mapping of JSON parser events to callback C types and value converters.
# Used to define the Callbacks structure and actual callback functions
# inside the parse function.
('null', C_EMPTY, lambda: None),
('boolean', C_INT, lambda v: bool(v)),
# "integer" and "double" aren't actually yielded by yajl since "number"
# takes precedence if defined
('integer', C_LONG, lambda v, l: int(string_at(v, l))),
('double', C_DOUBLE, lambda v, l: float(string_at(v, l))),
('number', C_STR, lambda v, l: common.number(b2s(string_at(v, l)))),
('string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
('start_map', C_EMPTY, lambda: None),
('map_key', C_STR, lambda v, l: b2s(string_at(v, l))),
('end_map', C_EMPTY, lambda: None),
('start_array', C_EMPTY, lambda: None),
('end_array', C_EMPTY, lambda: None),
]
class Callbacks(Structure):
_fields_ = [(name, type) for name, type, func in _callback_data]
class Config(Structure):
_fields_ = [
("allowComments", c_uint),
("checkUTF8", c_uint)
def test_common_number_is_deprecated(self):
with warning_catcher() as warns:
common.number("1")
self.assertEqual(len(warns), 1)
self.assertEqual(DeprecationWarning, warns[0].category)
def parse_value(target, multivalue):
"""
Parses results coming out of the Lexer into ijson events, which are sent to
`target`. A stack keeps track of the type of object being parsed at the time
(a value, and object or array -- the last two being values themselves).
A special EOF result coming from the Lexer indicates that no more content is
expected. This is used to check for incomplete content and raise the
appropriate exception, which wouldn't be possible if the Lexer simply closed
this co-routine (either explicitly via .close(), or implicitly by itself
finishing and decreasing the only reference to the co-routine) since that
causes a GeneratorExit exception that cannot be replaced with a custom one.
"""
state_stack = [_PARSE_VALUE]
pop = state_stack.pop
push = state_stack.append
send = target.send
prev_pos, prev_symbol = None, None
while True:
if prev_pos is None:
pos, symbol = (yield)
if (pos, symbol) == EOF:
if state_stack:
raise common.IncompleteJSONError('Incomplete JSON content')
break
else:
pos, symbol = prev_pos, prev_symbol
prev_pos, prev_symbol = None, None
try:
state = state_stack[-1]
except IndexError:
if multivalue:
state = _PARSE_VALUE
push(state)
else:
raise common.JSONError('Additional data found')
if state == _PARSE_VALUE:
# Simple, common cases
if symbol == 'null':
send(('null', None))
pop()
elif symbol == 'true':
send(('boolean', True))
pop()
elif symbol == 'false':
send(('boolean', False))
pop()
elif symbol[0] == '"':
send(('string', parse_string(symbol)))
pop()
# Array start
elif symbol == '[':
send(('start_array', None))
pos, symbol = (yield)
if (pos, symbol) == EOF:
if state_stack:
raise common.IncompleteJSONError(
'Incomplete JSON content')
break
if symbol == ']':
send(('end_array', None))
pop()
else:
prev_pos, prev_symbol = pos, symbol
push(_PARSE_ARRAY_ELEMENT_END)
push(_PARSE_VALUE)
# Object start
elif symbol == '{':
send(('start_map', None))
pos, symbol = (yield)
if (pos, symbol) == EOF:
if state_stack:
raise common.IncompleteJSONError(
'Incomplete JSON content')
break
if symbol == '}':
send(('end_map', None))
pop()
else:
prev_pos, prev_symbol = pos, symbol
push(_PARSE_OBJECT_KEY)
# A number
else:
try:
number = common.number(symbol)
except:
raise UnexpectedSymbol(symbol, pos)
else:
send(('number', number))
pop()
elif state == _PARSE_OBJECT_KEY:
if symbol[0] != '"':
raise UnexpectedSymbol(symbol, pos)
send(('map_key', parse_string(symbol)))
pos, symbol = (yield)
if (pos, symbol) == EOF:
if state_stack:
raise common.IncompleteJSONError('Incomplete JSON content')
break
if symbol != ':':
raise UnexpectedSymbol(symbol, pos)
state_stack[-1] = _PARSE_OBJECT_END
push(_PARSE_VALUE)
elif state == _PARSE_OBJECT_END:
if symbol == ',':
state_stack[-1] = _PARSE_OBJECT_KEY
elif symbol != '}':
raise UnexpectedSymbol(symbol, pos)
else:
send(('end_map', None))
pop()
pop()
elif state == _PARSE_ARRAY_ELEMENT_END:
if symbol == ',':
state_stack[-1] = _PARSE_ARRAY_ELEMENT_END
push(_PARSE_VALUE)
elif symbol != ']':
raise UnexpectedSymbol(symbol, pos)
else:
send(('end_array', None))
pop()
pop()
