def _parse_items_multiline(tokens, offset, endtoken, parse_item):
more_head = ()
more_tail = ()
items = []
while True:
head, offset = get_pattern(tokens, offset, PT_HEAD)
# It's possible that there's comments / newlines after the last
# item. In that case, we augment the tail of the previous item.
# If there are no items, this augments the head of the list itself.
if matches_pattern(tokens, offset, endtoken):
if head and isinstance(head[-1], ast.Indent):
head, more_tail = head[:-1], head[-1:]
if items:
items[-1] = items[-1]._replace(tail=items[-1].tail + head)
else:
more_head = head
break
val, offset = parse_item(tokens, offset, head=head)
# Allow multiple items to be on a single line
while not matches_pattern(tokens, offset, PT_COMMA_REST_OF_LINE):
rest, offset = get_pattern(tokens, offset, PT_COMMA_SPACE)
val = val._replace(tail=val.tail + rest)
items.append(val)
val, offset = parse_item(tokens, offset, head=())
rest, offset = get_pattern(tokens, offset, PT_COMMA_REST_OF_LINE)
val = val._replace(tail=val.tail + rest)
items.append(val)
return tuple(items), more_head, more_tail, offset
def _parse_top_level_map(tokens, offset):
items = []
while True:
head, offset = get_pattern(tokens, offset, PT_HEAD)
if matches_pattern(tokens, offset, ast.EOF):
items[-1] = items[-1]._replace(tail=items[-1].tail + head)
break
item, offset = _parse_map_item(tokens, offset, head)
if matches_pattern(tokens, offset, PT_REST_OF_LINE):
tail, offset = get_pattern(tokens, offset, PT_REST_OF_LINE)
item = item._replace(tail=tail)
items.append(item)
return ast.Map(head=(), items=tuple(items), tail=()), offset
def _parse_container(tokens, offset, cls, starttoken, endtoken, parse_item):
head, offset, multiline = _parse_start(tokens, offset, starttoken)
func = _parse_items_multiline if multiline else _parse_items
itemsret = func(tokens, offset, endtoken, parse_item)
items, more_head, more_tail, offset = itemsret
tail, offset = get_pattern(tokens, offset, endtoken)
return cls(head + more_head, items, more_tail + tail), offset
def _parse_start(tokens, offset, ast_start):
ret, offset = get_pattern(tokens, offset, ast_start)
match = matches_pattern(tokens, offset, PT_REST_OF_LINE)
if match:
rest, offset = match.match()
ret += rest
return ret, offset, bool(match)
def _parse_items(tokens, offset, endtoken, parse_item):
items = []
while not matches_pattern(tokens, offset, endtoken):
val, offset = parse_item(tokens, offset, head=())
if not matches_pattern(tokens, offset, endtoken):
comma_space, offset = get_pattern(tokens, offset, PT_COMMA_SPACE)
val = val._replace(tail=val.tail + comma_space)
items.append(val)
return tuple(items), (), (), offset
def _parse_val(tokens, offset):
if matches_pattern(tokens, offset, PT_VALUE_TOKENS):
return get_pattern(tokens, offset, PT_VALUE_TOKENS, single=True)
elif matches_pattern(tokens, offset, ast.ListStart):
return _parse_list(tokens, offset)
elif matches_pattern(tokens, offset, ast.MapStart):
return _parse_map(tokens, offset)
else:
missing_pattern = Or(PT_VALUE_TOKENS, ast.ListStart, ast.MapStart)
pattern_expected(tokens, offset, missing_pattern)
def _parse_map_item(tokens, offset, head):
key, offset = get_pattern(tokens, offset, PT_KEY, single=True)
colon_space, offset = get_pattern(tokens, offset, PT_COLON_SPACE)
val, offset = _parse_val(tokens, offset)
return ast.MapItem(head, key, colon_space, val, ()), offset
def parse_from_tokens(tokens, offset=0):
head, offset = get_pattern(tokens, offset, PT_HEAD)
val, offset = _parse_top_level(tokens, offset)
tail, offset = _parse_eof(tokens, offset)
return ast.Doc(head, val, tail)
def _parse_eof(tokens, offset):
"""Parse the end of the file"""
ret, offset = get_pattern(tokens, offset, Star(PT_REST_OF_LINE))
get_pattern(tokens, offset, ast.EOF)
return ret, offset