def test_tokenize_floats():
token = tokenize_yaml(YAML_FLOATS)
expected = ListToken([
ScalarToken(100.0, 3, 7),
ScalarToken(100.0, 11, 16),
], 1, 17)
assert token == expected
def test_tokenize_list():
token = tokenize_yaml(YAML_LIST)
expected = ListToken([
ScalarToken(True, 3, 6),
ScalarToken(False, 10, 14),
ScalarToken(None, 18, 21),
], 1, 22)
assert token == expected
def test_tokenize_object():
token = tokenize_yaml(YAML_OBJECT)
expected = DictToken({
ScalarToken('a', 1, 1): ListToken([
ScalarToken(1, 8, 8),
ScalarToken(2, 14, 14),
ScalarToken(3, 20, 20)
], 6, 21),
ScalarToken('b', 22, 22): ScalarToken('test', 25, 30)
}, 1, 31)
assert token == expected
def test_tokenize_floats():
token = tokenize_json('[100.0, 1.0E+2, 1E+2]')
expected = ListToken([
ScalarToken(100.0, 1, 5),
ScalarToken(100.0, 8, 13),
ScalarToken(100.0, 16, 19),
], 0, 20)
assert token == expected
assert token.get_value() == [100.0, 1.0E+2, 1E+2]
assert token[0].get_value() == 100.0
assert token[0].start_index == 1
assert token[0].end_index == 5
def test_tokenize_list():
token = tokenize_json('[true, false, null]')
expected = ListToken([
ScalarToken(True, 1, 4),
ScalarToken(False, 7, 11),
ScalarToken(None, 14, 17),
], 0, 18)
assert token == expected
assert token.get_value() == [True, False, None]
assert token[0].get_value()
assert token[0].start_index == 1
assert token[0].end_index == 4
def test_tokenize_object():
token = tokenize_json('{"a": [1, 2, 3], "b": "test"}')
expected = DictToken({
ScalarToken('a', 1, 3): ListToken([
ScalarToken(1, 7, 7),
ScalarToken(2, 10, 10),
ScalarToken(3, 13, 13)
], 6, 14),
ScalarToken('b', 17, 19): ScalarToken('test', 22, 27)
}, 0, 28)
assert token == expected
assert token.get_value() == {"a": [1, 2, 3], "b": "test"}
assert token["a"].get_value() == [1, 2, 3]
assert token["a"].start_index == 6
assert token["a"].end_index == 14
assert token["a"].start.line_no == 1
assert token["a"].start.column_no == 7
assert token.get_key("a").get_value() == "a"
assert token.get_key("a").start_index == 1
assert token.get_key("a").end_index == 3
def _scan_once(string, idx):
try:
nextchar = string[idx]
except IndexError:
raise StopIteration(idx) from None
if nextchar == '"':
value, end = parse_string(string, idx + 1, strict)
return ScalarToken(value, idx, end - 1, content), end
elif nextchar == '{':
value, end = parse_object((string, idx + 1), strict, _scan_once,
memo, content)
return DictToken(value, idx, end - 1, content), end
elif nextchar == '[':
value, end = parse_array((string, idx + 1), _scan_once)
return ListToken(value, idx, end - 1, content), end
elif nextchar == 'n' and string[idx:idx + 4] == 'null':
value, end = None, idx + 4
return ScalarToken(value, idx, end - 1, content), end
elif nextchar == 't' and string[idx:idx + 4] == 'true':
value, end = True, idx + 4
return ScalarToken(value, idx, end - 1, content), end
elif nextchar == 'f' and string[idx:idx + 5] == 'false':
value, end = False, idx + 5
return ScalarToken(value, idx, end - 1, content), end
m = match_number(string, idx)
if m is not None:
integer, frac, exp = m.groups()
if frac or exp:
res = parse_float(integer + (frac or '') + (exp or ''))
else:
res = parse_int(integer)
value, end = res, m.end()
return ScalarToken(value, idx, end - 1, content), end
else:
raise StopIteration(idx)
def construct_sequence(loader, node):
start = node.start_mark.index
end = node.end_mark.index
value = loader.construct_sequence(node)
return ListToken(value, start, end - 1, content=content)