def tokenize_json(content):
assert isinstance(content, (str, bytes))
if isinstance(content, bytes):
content = content.decode('utf-8', 'ignore')
if not content.strip():
message = ErrorMessage(text='No content.',
code='parse_error',
position=Position(line_no=1,
column_no=1,
index=0))
raise ParseError(messages=[message], summary='Invalid JSON.')
try:
decoder = _TokenizingDecoder(content=content)
return decoder.decode(content)
except json.decoder.JSONDecodeError as exc:
message = ErrorMessage(
text=_strip_endings(exc.msg, [" starting at", " at"]) + ".",
code='parse_error',
position=Position(line_no=exc.lineno,
column_no=exc.colno,
index=exc.pos))
raise ParseError(messages=[message], summary='Invalid JSON.') from None
def test_invalid_properties():
with pytest.raises(ValidationError) as exc:
apistar.parse('{"a": "abc", "b": 123}', encoding="json", validator=VALIDATOR)
assert exc.value.messages == [
ErrorMessage(
text='Must be a number.',
code='type',
index=['a'],
position=Position(line_no=1, column_no=7, index=6)
),
ErrorMessage(
text='Invalid property name.',
code='invalid_property',
index=['b'],
position=Position(line_no=1, column_no=14, index=13)
)
]
def test_object_unterminated_after_value():
with pytest.raises(ParseError) as exc:
apistar.parse('{"abc": "def"', encoding='json')
assert exc.value.messages == [
ErrorMessage(
text="Expecting ',' delimiter.",
code='parse_error',
position=Position(line_no=1, column_no=14, index=13)
)
]
def test_object_invalid_property_name():
with pytest.raises(ParseError) as exc:
apistar.parse('{"abc": "def", 1', encoding='json')
assert exc.value.messages == [
ErrorMessage(
text="Expecting property name enclosed in double quotes.",
code='parse_error',
position=Position(line_no=1, column_no=16, index=15)
)
]
def test_object_missing_comma_delimiter():
with pytest.raises(ParseError) as exc:
apistar.parse('{"abc": "def" 1', encoding='json')
assert exc.value.messages == [
ErrorMessage(
text="Expecting ',' delimiter.",
code='parse_error',
position=Position(line_no=1, column_no=15, index=14)
)
]
def test_object_missing_property_name():
with pytest.raises(ParseError) as exc:
apistar.parse('{', encoding='json')
assert exc.value.messages == [
ErrorMessage(
text='Expecting property name enclosed in double quotes.',
code='parse_error',
position=Position(line_no=1, column_no=2, index=1)
)
]
def test_empty_string():
with pytest.raises(ParseError) as exc:
apistar.parse(b'', encoding='json')
assert exc.value.messages == [
ErrorMessage(
text='No content.',
code='parse_error',
position=Position(line_no=1, column_no=1, index=0)
)
]
def test_missing_required_property():
with pytest.raises(ValidationError) as exc:
apistar.parse('{}', encoding="json", validator=VALIDATOR)
assert exc.value.messages == [
ErrorMessage(
text='The "a" field is required.',
code='required',
index=['a'],
position=Position(line_no=1, column_no=1, index=0))
]
def test_unterminated_string():
with pytest.raises(ParseError) as exc:
apistar.parse('"ab', encoding='json')
assert exc.value.messages == [
ErrorMessage(
text="Unterminated string.",
code='parse_error',
position=Position(line_no=1, column_no=1, index=0)
)
]
def test_invalid_token():
with pytest.raises(ParseError) as exc:
apistar.parse('-', encoding='json')
assert exc.value.messages == [
ErrorMessage(
text="Expecting value.",
code='parse_error',
position=Position(line_no=1, column_no=1, index=0)
)
]
def test_invalid_top_level_item():
with pytest.raises(ValidationError) as exc:
apistar.parse('123', encoding="json", validator=VALIDATOR)
assert exc.value.messages == [
ErrorMessage(
text='Must be an object.',
code='type',
index=None,
position=Position(line_no=1, column_no=1, index=0)
)
]
def _get_position(content, index):
return Position(line_no=content.count('\n', 0, index) + 1,
column_no=index - content.rfind('\n', 0, index),
index=index)
def tokenize_yaml(content):
class CustomLoader(SafeLoader):
pass
def construct_mapping(loader, node):
start = node.start_mark.index
end = node.end_mark.index
mapping = loader.construct_mapping(node)
return DictToken(mapping, start, end - 1, content=content)
def construct_sequence(loader, node):
start = node.start_mark.index
end = node.end_mark.index
value = loader.construct_sequence(node)
return ListToken(value, start, end - 1, content=content)
def construct_scalar(loader, node):
start = node.start_mark.index
end = node.end_mark.index
value = loader.construct_scalar(node)
return ScalarToken(value, start, end - 1, content=content)
def construct_int(loader, node):
start = node.start_mark.index
end = node.end_mark.index
value = loader.construct_yaml_int(node)
return ScalarToken(value, start, end - 1, content=content)
def construct_float(loader, node):
start = node.start_mark.index
end = node.end_mark.index
value = loader.construct_yaml_float(node)
return ScalarToken(value, start, end - 1, content=content)
def construct_bool(loader, node):
start = node.start_mark.index
end = node.end_mark.index
value = loader.construct_yaml_bool(node)
return ScalarToken(value, start, end - 1, content=content)
def construct_null(loader, node):
start = node.start_mark.index
end = node.end_mark.index
value = loader.construct_yaml_null(node)
return ScalarToken(value, start, end - 1, content=content)
CustomLoader.add_constructor(
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
CustomLoader.add_constructor(
yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, construct_sequence)
CustomLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG,
construct_scalar)
CustomLoader.add_constructor('tag:yaml.org,2002:int', construct_int)
CustomLoader.add_constructor('tag:yaml.org,2002:float', construct_float)
CustomLoader.add_constructor('tag:yaml.org,2002:bool', construct_bool)
CustomLoader.add_constructor('tag:yaml.org,2002:null', construct_null)
assert isinstance(content, (str, bytes))
if isinstance(content, bytes):
content = content.decode('utf-8', 'ignore')
if not content.strip():
message = ErrorMessage(text='No content.',
code='parse_error',
position=Position(line_no=1,
column_no=1,
index=0))
raise ParseError(errors=[message], summary='Invalid YAML.')
try:
return yaml.load(content, CustomLoader)
except (yaml.scanner.ScannerError, yaml.parser.ParserError) as exc:
index = getattr(exc, 'index', 0)
message = ErrorMessage(text=exc.problem + ".",
code='parse_error',
position=_get_position(content, index=index))
raise ParseError(messages=[message], summary='Invalid YAML.') from None
def _get_position(self, index):
content = self._content[:index + 1]
lines = content.splitlines()
line_no = max(len(lines), 1)
column_no = 1 if not lines else max(len(lines[-1]), 1)
return Position(line_no, column_no, index)